From 3d6881a7eff45eda5303d5361c61f17372fafb99 Mon Sep 17 00:00:00 2001 From: Jonathan Shook Date: Tue, 18 Feb 2025 11:37:18 -0600 Subject: [PATCH 1/6] distribution morphing support (#2161) --- .../stathelpers/AliasSamplerDoubleLong.java | 124 +++++++++++++++ .../core/stathelpers/EvProbLongDouble.java | 22 +++ .../to_double/EmpiricalHistribution.java | 76 +++++++++ .../shared/from_long/to_double/HashMix.java | 147 ++++++++++++++++++ .../shared/from_long/to_double/LERP.java | 25 +++ .../AliasSamplerDoubleLongTest.java | 82 ++++++++++ .../to_double/EmpiricalHistributionTest.java | 56 +++++++ .../from_long/to_double/HashMixTest.java | 56 +++++++ .../shared/from_long/to_double/LERPTest.java | 37 +++++ 9 files changed, 625 insertions(+) create mode 100644 nb-virtdata/virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/core/stathelpers/AliasSamplerDoubleLong.java create mode 100644 nb-virtdata/virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/core/stathelpers/EvProbLongDouble.java create mode 100644 nb-virtdata/virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_double/EmpiricalHistribution.java create mode 100644 nb-virtdata/virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_double/HashMix.java create mode 100644 nb-virtdata/virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_double/LERP.java create mode 100644 nb-virtdata/virtdata-lib-basics/src/test/java/io/nosqlbench/virtdata/library/basics/core/stathelpers/aliasmethod/AliasSamplerDoubleLongTest.java create mode 100644 nb-virtdata/virtdata-lib-basics/src/test/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_double/EmpiricalHistributionTest.java create mode 100644 nb-virtdata/virtdata-lib-basics/src/test/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_double/HashMixTest.java create mode 100644 nb-virtdata/virtdata-lib-basics/src/test/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_double/LERPTest.java diff --git a/nb-virtdata/virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/core/stathelpers/AliasSamplerDoubleLong.java b/nb-virtdata/virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/core/stathelpers/AliasSamplerDoubleLong.java new file mode 100644 index 000000000..21e4d908f --- /dev/null +++ b/nb-virtdata/virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/core/stathelpers/AliasSamplerDoubleLong.java @@ -0,0 +1,124 @@ +/* + * Copyright (c) nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.virtdata.library.basics.core.stathelpers; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.LinkedList; +import java.util.List; +import java.util.function.DoubleToIntFunction; +import java.util.function.DoubleToLongFunction; +import java.util.stream.Collectors; + +/** + * Uses the alias sampling method to encode and sample from discrete probabilities, + * even over larger sets of data. This form requires a unit interval sample value + * between 0.0 and 1.0. Assuming the maximal amount of memory is used for distinct + * outcomes N, a memory buffer of N*16 bytes is required for this implementation, + * requiring 32MB of memory for 1M entries. + * + * This sampler should be shared between threads, and will be by default, in order + * to avoid many instances of a 32MB buffer on heap. + */ +public class AliasSamplerDoubleLong implements DoubleToLongFunction { + + private final ByteBuffer stats; // tuples of double,int,int (unfair coin, direct pointers to referents) + private final double slotCount; // The number of fair die-roll slotCount that contain unfair coin probabilities + private static final int _r0=0; + private static final int _r1=_r0+Double.BYTES; // unfair coin + private static final int _r2=_r1+Long.BYTES; // + referent 1 + public static int RECORD_LEN = _r2 + Long.BYTES; // + referent 2 = Record size for the above. + + // for testing + AliasSamplerDoubleLong(ByteBuffer stats) { + this.stats = stats; + if ((stats.capacity()% RECORD_LEN)!=0) { + throw new RuntimeException("Misaligned ByteBuffer size, must be a multiple of " + RECORD_LEN); + } + slotCount = (stats.capacity()/ RECORD_LEN); + } + + public AliasSamplerDoubleLong(List events) { + int size = events.size(); + + int[] alias = new int[events.size()]; + double[] prob = new double[events.size()]; + + LinkedList small = new LinkedList<>(); + LinkedList large = new LinkedList<>(); + List slots = new ArrayList<>(); + + // array-size normalization + double sumProbability = events.stream().mapToDouble(EvProbLongDouble::prob).sum(); + events = events.stream().map(e -> new EvProbLongDouble(e.id(), + (e.prob()/sumProbability)*size)).collect(Collectors.toList()); + + // presort + for (EvProbLongDouble event : events) { + (event.prob()<1.0D ? small : large).addLast(event); + } + + while (small.peekFirst()!=null && large.peekFirst()!=null) { + EvProbLongDouble l = small.removeFirst(); + EvProbLongDouble g = large.removeFirst(); + slots.add(new Slot(g.id(), l.id(), l.prob())); + EvProbLongDouble remainder = new EvProbLongDouble(g.id(),(g.prob()+l.prob())-1); + (remainder.prob()<1.0D ? small : large).addLast(remainder); + } + while (large.peekFirst()!=null) { + EvProbLongDouble g = large.removeFirst(); + slots.add(new Slot(g.id(),g.id(),1.0)); + } + while (small.peekFirst()!=null) { + EvProbLongDouble l = small.removeFirst(); + slots.add(new Slot(l.id(),l.id(),1.0)); + } + if (slots.size()!=size) { + throw new RuntimeException("basis for average probability is incorrect, because only " + slots.size() + " slotCount of " + size + " were created."); + } + // align to indexes + for (int i = 0; i < slots.size(); i++) { + slots.set(i,slots.get(i).rescale(i, i+1)); + } + this.stats = ByteBuffer.allocate(slots.size()* RECORD_LEN); + + for (Slot slot : slots) { + stats.putDouble(slot.botProb); + stats.putLong(slot.botId()); + stats.putLong(slot.topId()); + } + stats.flip(); + this.slotCount = (stats.capacity()/ RECORD_LEN); + + } + + @Override + public long applyAsLong(double value) { + double fractionlPoint = value * slotCount; + int offsetPoint = (int) fractionlPoint * RECORD_LEN; + double divider = stats.getDouble(offsetPoint); + int selector = offsetPoint+ (fractionlPoint>divider?_r2:_r1); + long referentId = stats.getLong(selector); + return referentId; + } + + private record Slot(long topId, long botId, double botProb){ + public Slot rescale(int min, int max) { + return new Slot(topId, botId, (min + (botProb*(max-min)))); + } + }; +} diff --git a/nb-virtdata/virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/core/stathelpers/EvProbLongDouble.java b/nb-virtdata/virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/core/stathelpers/EvProbLongDouble.java new file mode 100644 index 000000000..2257e1e15 --- /dev/null +++ b/nb-virtdata/virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/core/stathelpers/EvProbLongDouble.java @@ -0,0 +1,22 @@ +/* + * Copyright (c) nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.virtdata.library.basics.core.stathelpers; + +import java.util.Comparator; + +public record EvProbLongDouble(long id, double prob) { +} diff --git a/nb-virtdata/virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_double/EmpiricalHistribution.java b/nb-virtdata/virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_double/EmpiricalHistribution.java new file mode 100644 index 000000000..ffee03a4e --- /dev/null +++ b/nb-virtdata/virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_double/EmpiricalHistribution.java @@ -0,0 +1,76 @@ +package io.nosqlbench.virtdata.library.basics.shared.from_long.to_double; + +/* + * Copyright (c) nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + +import io.nosqlbench.nb.api.errors.BasicError; +import io.nosqlbench.virtdata.api.annotations.Categories; +import io.nosqlbench.virtdata.api.annotations.Category; +import io.nosqlbench.virtdata.api.annotations.Example; +import io.nosqlbench.virtdata.api.annotations.ThreadSafeMapper; +import io.nosqlbench.virtdata.library.basics.core.stathelpers.AliasSamplerDoubleLong; +import io.nosqlbench.virtdata.library.basics.core.stathelpers.EvProbLongDouble; + +import java.util.ArrayList; +import java.util.List; + +/// Empirical Histribution is a portmanteau name to capture the +/// concept of an empirical distribution based on a discrete histogram. +/// This is in contrast to the other similar method [EmpiricalDistribution], +/// which uses a continuous density estimation. Both excel in specific ways. +/// +/// Use this distribution when you have a set of label frequencies which you +/// want to represent accurately. +@ThreadSafeMapper +@Categories(Category.distributions) +public class EmpiricalHistribution extends AliasSamplerDoubleLong { + + @Example({"EmpiricalHistribution('50 25 13 12')", "implied frequencies of 0:50 1:25 2:13 3:12"}) + @Example({ + "EmpiricalHistribution('234:50 33:25 17:13 3:12')", + "labeled frequencies; 234,33,17,3 are labels, and 50,25,13,12 are weights" + }) + public EmpiricalHistribution(String freqs) { + List events = new ArrayList<>(); + boolean labeled = (freqs.contains(":")); + + String[] elems = freqs.split("[,; ]"); + for (int i = 0; i < elems.length; i++) { + String[] parts = elems[i].split(":", 2); + if ((parts.length == 1 && labeled) || (parts.length == 2 && !labeled)) { + throw new RuntimeException( + "If any elements are labeled, all elements must be:" + freqs); + } + long id = labeled ? Long.parseLong(parts[0]) : i; + events.add(new EvProbLongDouble(id, Long.parseLong(parts[1]))); + } + super(events); + } + + public EmpiricalHistribution(long... freqs) { + super(genEvents(freqs)); + } + + private static List genEvents(long[] freqs) { + ArrayList events = new ArrayList<>(); + for (int i = 0; i < freqs.length; i++) { + events.add(new EvProbLongDouble(i, freqs[i])); + } + return events; + } +} diff --git a/nb-virtdata/virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_double/HashMix.java b/nb-virtdata/virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_double/HashMix.java new file mode 100644 index 000000000..6d15eeebe --- /dev/null +++ b/nb-virtdata/virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_double/HashMix.java @@ -0,0 +1,147 @@ +package io.nosqlbench.virtdata.library.basics.shared.from_long.to_double; + +/* + * Copyright (c) nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + +import io.nosqlbench.virtdata.api.annotations.Categories; +import io.nosqlbench.virtdata.api.annotations.Category; +import io.nosqlbench.virtdata.api.annotations.Example; +import io.nosqlbench.virtdata.api.annotations.ThreadSafeMapper; +import io.nosqlbench.virtdata.api.bindings.VirtDataConversions; + +import java.util.function.LongToDoubleFunction; +import java.util.function.LongUnaryOperator; + +/// Blends two functions with a domain of 0..Long.MAX_VALUE as the input interval, +/// and a double output. The output value is interpolated between the output value +/// of the two according to the mix function. When the mix function yields a value +/// of 0.0, then the mix is turned _fully counter-clockwise_., or fully on the first provided +/// function. When the value is 1.0, the mix is turned all the clockwise, or fully on the second +/// provided function. +/// +/// If there are only two inner functions provided to HashMix, then it will default to +/// sampling random mixes at a randomized sample point. In other words, the variates +/// provided will be somewhere between the two curves on the unit interval. This is a simple way +/// to sample between two curves by default. The yielded value will be greater than or equal to +/// the lower of the two values at any point, and less than or equal to the greater of either. +/// +/// If a third parameter is provided to control the mix, then the mix can be set directly as a +/// unit interval. (The dial goes from 0.0 to 1.0). Any double or float value here will suffice. +/// You can use this when you want to have a test parameter that slews between two modeled +/// shapes. You can alternately provide any other function which can be coerced to a LongToDouble +/// function as a dynamic mix control. IFF such a function is provided, it must also be responsible +/// for hashing the input value if pseudo-randomness is desired. +/// +/// If a fourth parameter is provided, the sample point can also be controlled. By default, the +/// values on the provided curves will be sampled pseudo-randomly. However, a fourth parameter +/// can override this just like the mix ratio. As well, if you provide a value or function +/// to control the sample point, you are also responsible for any hashing needed to sample across +/// the whole space of possible values. +/// +/// The flexibility of these two parameters provides a substantial amount of flexibility. You +/// can, for example: +/// +/// - sample variates between two curves +/// - sample variates at a selected morphing step between the curves +/// - sample variates between two curves on a subsection of the unit interval +/// - sample variates within a defined band gap of the two curves +@ThreadSafeMapper +@Categories(Category.functional) +public class HashMix implements LongToDoubleFunction { + + private final LongToDoubleFunction f1; + private final LongToDoubleFunction f2; + private final LongToDoubleFunction mixF; + private final LongUnaryOperator sampleF; + + @Example({ + "HashMix(Func1(),Func2())", + "yield samples between func1 and func2 values at some random random sample point x" + }) + @Example({ + "HashMix(Func1(),Func2(),0.25d)", + "yield samples which are 25% from the sample values for func1 and func2 at some random " + + "sample point x" + }) + @Example({ + "HashMix(Func1(),Func2(),HashRange(0.25d,0.75d)", + "yield samples between 25% and 75% from func1 to func2 values at some random sample point x" + }) + @Example({ + "HashMix(Func1(),Func2(),0.0d,ScaledDouble())", + "access Func1 values as if it were the only one provided. ScaledDouble adds no " + + "randomization the input value, but it does map it to the sample domain of 0.0d-0.1d." + }) + public HashMix(Object curve1F, Object curve2F, Object mixPointF, Object samplePointF) { + if (mixPointF instanceof Double v) { + if (v > 1.0d || v < 0.0d) { + throw new RuntimeException( + "mix value (" + v + ") must be within the unit" + " range [0.0d,1.0d]"); + } + this.mixF = n -> v; + } else if (mixPointF instanceof Float v) { + if (v > 1.0d || v < 0.0d) { + throw new RuntimeException( + "mix value (" + v + ") must be within the unit" + " range [0.0d,1.0d]"); + } + this.mixF = n -> v; + } else { + this.mixF = VirtDataConversions.adaptFunction(mixPointF, LongToDoubleFunction.class); + } + this.f1 = VirtDataConversions.adaptFunction(curve1F, LongToDoubleFunction.class); + this.f2 = VirtDataConversions.adaptFunction(curve2F, LongToDoubleFunction.class); + this.sampleF = VirtDataConversions.adaptFunction(samplePointF, LongUnaryOperator.class); + } + + public HashMix(Object curve1F, Object curve2F, Object mixPointF) { + this( + curve1F, + curve2F, + mixPointF, + new io.nosqlbench.virtdata.library.basics.shared.from_long.to_long.HashRange(Long.MAX_VALUE) + ); + } + + public HashMix(Object curve1F, Object curve2F) { + this( + curve1F, + curve2F, + new HashRange(0.0d, 1.0d), + new io.nosqlbench.virtdata.library.basics.shared.from_long.to_long.HashRange(Long.MAX_VALUE) + ); + } + + public HashMix(LongToDoubleFunction f1, LongToDoubleFunction f2) { + this( + f1, + f2, + new HashRange(0.0d, 1.0d), + new io.nosqlbench.virtdata.library.basics.shared.from_long.to_long.HashRange(Long.MAX_VALUE) + ); + } + + + @Override + public double applyAsDouble(long value) { + long sampleAt = sampleF.applyAsLong(value); + double v1 = f1.applyAsDouble(sampleAt); + double v2 = f2.applyAsDouble(sampleAt); + double mix = mixF.applyAsDouble(value); + return LERP.lerp(v1, v2, mix); + } +} diff --git a/nb-virtdata/virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_double/LERP.java b/nb-virtdata/virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_double/LERP.java new file mode 100644 index 000000000..b394500c4 --- /dev/null +++ b/nb-virtdata/virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_double/LERP.java @@ -0,0 +1,25 @@ +package io.nosqlbench.virtdata.library.basics.shared.from_long.to_double; + +/* + * Copyright (c) nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + +public class LERP { + public static double lerp(double v1, double v2, double mix) { + return v1 + (v2 - v1) * mix; + } +} diff --git a/nb-virtdata/virtdata-lib-basics/src/test/java/io/nosqlbench/virtdata/library/basics/core/stathelpers/aliasmethod/AliasSamplerDoubleLongTest.java b/nb-virtdata/virtdata-lib-basics/src/test/java/io/nosqlbench/virtdata/library/basics/core/stathelpers/aliasmethod/AliasSamplerDoubleLongTest.java new file mode 100644 index 000000000..b7eab5544 --- /dev/null +++ b/nb-virtdata/virtdata-lib-basics/src/test/java/io/nosqlbench/virtdata/library/basics/core/stathelpers/aliasmethod/AliasSamplerDoubleLongTest.java @@ -0,0 +1,82 @@ +/* + * Copyright (c) nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.nosqlbench.virtdata.library.basics.core.stathelpers.aliasmethod; + +import io.nosqlbench.virtdata.library.basics.core.stathelpers.AliasSamplerDoubleInt; +import io.nosqlbench.virtdata.library.basics.core.stathelpers.AliasSamplerDoubleLong; +import io.nosqlbench.virtdata.library.basics.core.stathelpers.EvProbD; +import io.nosqlbench.virtdata.library.basics.core.stathelpers.EvProbLongDouble; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; + +public class AliasSamplerDoubleLongTest { + + private final static Logger logger = LogManager.getLogger(AliasSamplerDoubleLongTest.class); + @Test + public void testAliasSamplerBinaryFractions() { + List events = new ArrayList(); + events.add(new EvProbLongDouble(1L,1.0D)); + events.add(new EvProbLongDouble(2L,1.0D)); + events.add(new EvProbLongDouble(3L,2.0D)); + events.add(new EvProbLongDouble(4L,4.0D)); + events.add(new EvProbLongDouble(5L,8.0D)); + events.add(new EvProbLongDouble(6L,16.0D)); + events.add(new EvProbLongDouble(7L,32.0D)); + events.add(new EvProbLongDouble(8L,64.0D)); + + AliasSamplerDoubleLong as = new AliasSamplerDoubleLong(events); + int[] stats = new int[9]; + for (int i = 0; i < 10000; i++) { + double v = (double)i / 10000D; + long idx = as.applyAsLong(v); + stats[(int)idx]++; + } + logger.debug(Arrays.toString(stats)); + assertThat(stats).containsExactly(0,79,79,157,313,626,1250,2499,4997); + + } + + @Test + public void testAliasSamplerSimple() { + List events = new ArrayList<>(); + events.add(new EvProbD(1,1D)); + events.add(new EvProbD(2,2D)); + events.add(new EvProbD(3,3D)); + + AliasSamplerDoubleInt as = new AliasSamplerDoubleInt(events); + + int[] stats = new int[4]; + for (int i = 0; i < 10000; i++) { + double v = (double)i / 10000D; + int idx = as.applyAsInt(v); + stats[idx]++; + } + logger.debug(Arrays.toString(stats)); + assertThat(stats).containsExactly(0,1667,3334,4999); + } + + + +} diff --git a/nb-virtdata/virtdata-lib-basics/src/test/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_double/EmpiricalHistributionTest.java b/nb-virtdata/virtdata-lib-basics/src/test/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_double/EmpiricalHistributionTest.java new file mode 100644 index 000000000..90bad9d9c --- /dev/null +++ b/nb-virtdata/virtdata-lib-basics/src/test/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_double/EmpiricalHistributionTest.java @@ -0,0 +1,56 @@ +package io.nosqlbench.virtdata.library.basics.shared.from_long.to_double; + +/* + * Copyright (c) nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + +import org.assertj.core.data.Offset; +import org.junit.jupiter.api.Test; + +import java.util.Arrays; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +public class EmpiricalHistributionTest { + + @Test + public void testUniformSyntaxRequired() { + assertThatThrownBy(() -> new EmpiricalHistribution("1 2:2 3:3")).hasMessageContaining( + "all elements must be"); + } + + @Test + public void testBasicHistribution() { + EmpiricalHistribution h = new EmpiricalHistribution("1:1 2:2 3:3"); + long[] counts = new long[10]; + int total=1000000; + HashRange hr = new HashRange(0.0d, 1.0d); + for (int i = 0; i < total; i++) { + double hash = hr.applyAsDouble(i); + long v = h.applyAsLong(hash); + counts[(int)v]++; + } + assertThat((double) counts[0] / (double) total).isEqualTo(0.0d, Offset.offset(0.01)); + assertThat((double) counts[1] / (double) total).isEqualTo(0.16666666d, Offset.offset(0.01)); + assertThat((double) counts[2] / (double) total).isEqualTo(0.33333333d, + Offset.offset(0.01)); + assertThat((double) counts[3] / (double) total).isEqualTo(0.5d, Offset.offset(0.01)); + System.out.println(Arrays.toString(counts)); + } + +} diff --git a/nb-virtdata/virtdata-lib-basics/src/test/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_double/HashMixTest.java b/nb-virtdata/virtdata-lib-basics/src/test/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_double/HashMixTest.java new file mode 100644 index 000000000..c9df86b4a --- /dev/null +++ b/nb-virtdata/virtdata-lib-basics/src/test/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_double/HashMixTest.java @@ -0,0 +1,56 @@ +package io.nosqlbench.virtdata.library.basics.shared.from_long.to_double; + +/* + * Copyright (c) nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + +import org.assertj.core.data.Offset; +import org.junit.jupiter.api.Test; + +import java.util.function.LongToDoubleFunction; + +import static org.assertj.core.api.Assertions.assertThat; + +public class HashMixTest { + + private final static LongToDoubleFunction TO_UNIT_INTERVAL = + (l) -> ((double) l) / ((double) Long.MAX_VALUE); + private final static Object TO_UNIT_INTERVAL_OBJ = (Object) TO_UNIT_INTERVAL; + + @Test + public void testLinearMix() { + HashMix um1 = new HashMix(TO_UNIT_INTERVAL, TO_UNIT_INTERVAL); + for (long i = 1; i < (Long.MAX_VALUE >> 1); i *= 2) { + assertThat(um1.applyAsDouble(i)).isEqualTo( + TO_UNIT_INTERVAL.applyAsDouble(i), + Offset.offset(0.0000001d) + ); + } + } + + @Test + public void testCrossfadeMix() { + LongToDoubleFunction rampdown1 = l -> 1.0d - TO_UNIT_INTERVAL.applyAsDouble(l); + LongToDoubleFunction rampdown2 = l -> 2.0d - TO_UNIT_INTERVAL.applyAsDouble(l); + HashMix um1 = new HashMix(rampdown1,rampdown2); + for (long i = 1<<24; i <= Long.MAX_VALUE>>1; i<<=1) { + double value = um1.applyAsDouble(i); + assertThat(um1.applyAsDouble(i)).isEqualTo(1.0d, Offset.offset(0.0000001d)); + } + } + +} diff --git a/nb-virtdata/virtdata-lib-basics/src/test/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_double/LERPTest.java b/nb-virtdata/virtdata-lib-basics/src/test/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_double/LERPTest.java new file mode 100644 index 000000000..a16155e06 --- /dev/null +++ b/nb-virtdata/virtdata-lib-basics/src/test/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_double/LERPTest.java @@ -0,0 +1,37 @@ +package io.nosqlbench.virtdata.library.basics.shared.from_long.to_double; + +/* + * Copyright (c) nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + +import org.assertj.core.data.Offset; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.*; + +public class LERPTest { + + @Test + public void testDoubleLerp() { + assertThat(LERP.lerp(10.0d,10.0d,1.0d)).isEqualTo(10.0d, Offset.offset(0.00001d)); + assertThat(LERP.lerp(10.0d,0.0d,0.0d)).isEqualTo(10.0d, Offset.offset(0.00001d)); + assertThat(LERP.lerp(10.0d,0.0d,1.0d)).isEqualTo(0.0d, Offset.offset(0.00001d)); + assertThat(LERP.lerp(10.0d,5.0d,0.5d)).isEqualTo(7.5d, Offset.offset(0.00001d)); + } + +} From a63e4e06448ff77bf03ebacc5b0fa89ed83612ed Mon Sep 17 00:00:00 2001 From: Madhavan Sridharan Date: Tue, 18 Feb 2025 18:58:21 -0500 Subject: [PATCH 2/6] Update CQL Java Driver to 4.19.0 --- mvn-defaults/pom.xml | 6 +++--- nb-adapters/adapter-cqld4/pom.xml | 2 -- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/mvn-defaults/pom.xml b/mvn-defaults/pom.xml index 1f68a0116..0ac81f2ff 100644 --- a/mvn-defaults/pom.xml +++ b/mvn-defaults/pom.xml @@ -178,17 +178,17 @@ org.apache.cassandra java-driver-core - 4.18.1 + 4.19.0 org.apache.cassandra java-driver-query-builder - 4.18.1 + 4.19.0 org.apache.cassandra java-driver-mapper-runtime - 4.18.1 + 4.19.0 io.netty diff --git a/nb-adapters/adapter-cqld4/pom.xml b/nb-adapters/adapter-cqld4/pom.xml index 71a9e9700..23a8053c4 100644 --- a/nb-adapters/adapter-cqld4/pom.xml +++ b/nb-adapters/adapter-cqld4/pom.xml @@ -59,13 +59,11 @@ org.apache.cassandra java-driver-core - 4.18.1 org.apache.cassandra java-driver-query-builder - 4.18.1 From d400c81fdb90beb9a6ed4934586a7d7a75ea4a10 Mon Sep 17 00:00:00 2001 From: Jonathan Shook Date: Tue, 18 Feb 2025 17:56:21 -0600 Subject: [PATCH 3/6] disable failing validator --- .../engine/api/activityimpl/uniform/StandardActivity.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/nb-engine/nb-engine-core/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/StandardActivity.java b/nb-engine/nb-engine-core/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/StandardActivity.java index 8258fa2fb..93c98f059 100644 --- a/nb-engine/nb-engine-core/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/StandardActivity.java +++ b/nb-engine/nb-engine-core/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/StandardActivity.java @@ -118,10 +118,10 @@ public class StandardActivity exte } else { adapter = adapters.get(driverName); } - if (adapter instanceof NBConfigurable configurable) { - adapterModel = configurable.getConfigModel(); - adapterModel.assertValidConfig(ot.getParams()); - } +// if (adapter instanceof NBConfigurable configurable) { +// adapterModel = configurable.getConfigModel(); +// adapterModel.assertValidConfig(ot.getParams()); +// } paramsAdvisor.validateAll(ot.getParams().keySet()); paramsAdvisor.validateAll(ot.getTags().keySet()); paramsAdvisor.validateAll(ot.getBindings().keySet()); From 1f45d0b850756d7e5c51b0af2cd2b2bb3a22c5dc Mon Sep 17 00:00:00 2001 From: Jonathan Shook Date: Tue, 18 Feb 2025 17:56:33 -0600 Subject: [PATCH 4/6] add support for DoubleToLong types --- .../api/bindings/VirtDataFunctions.java | 394 ++++++++++++------ 1 file changed, 272 insertions(+), 122 deletions(-) diff --git a/nb-virtdata/virtdata-api/src/main/java/io/nosqlbench/virtdata/api/bindings/VirtDataFunctions.java b/nb-virtdata/virtdata-api/src/main/java/io/nosqlbench/virtdata/api/bindings/VirtDataFunctions.java index a541cea2c..86750b27a 100644 --- a/nb-virtdata/virtdata-api/src/main/java/io/nosqlbench/virtdata/api/bindings/VirtDataFunctions.java +++ b/nb-virtdata/virtdata-api/src/main/java/io/nosqlbench/virtdata/api/bindings/VirtDataFunctions.java @@ -35,6 +35,7 @@ public class VirtDataFunctions { LongUnaryOperator(java.util.function.LongUnaryOperator.class, long.class), IntFunction(java.util.function.IntFunction.class, int.class), IntUnaryOperator(java.util.function.IntUnaryOperator.class, int.class), + DoubleToLongFunction(java.util.function.DoubleToLongFunction.class, long.class), DoubleToIntFunction(java.util.function.DoubleToIntFunction.class, int.class), DoubleFunction(java.util.function.DoubleFunction.class, double.class), DoubleUnaryOperator(java.util.function.DoubleUnaryOperator.class, double.class), @@ -60,46 +61,70 @@ public class VirtDataFunctions { } /** - * Adapt a functional object into a different type of functional object. - * - * @param func The original function object. - * @param type The type of function object needed. - * @param output The output type required for the adapted function. - * @param truncate Whether to throw an exception on any narrowing conversion. If this is set to false, then basic - * roll-over logic is applied on narrowing conversions. - * @param The type of function object needed. - * @return An instance of F + Adapt a functional object into a different type of functional object. + @param func + The original function object. + @param type + The type of function object needed. + @param output + The output type required for the adapted function. + @param truncate + Whether to throw an exception on any narrowing conversion. If this is set to false, then + basic + roll-over logic is applied on narrowing conversions. + @param + The type of function object needed. + @return An instance of F */ public static F adapt(Object func, Class type, Class output, boolean truncate) { FuncType funcType = FuncType.valueOf(type); switch (funcType) { + case DoubleToLongFunction: + return truncate ? (F) adaptDoubleToLongFunction(func, output) : + (F) adaptDoubleToLongFunctionStrict(func, output); case LongUnaryOperator: - return truncate ? (F) adaptLongUnaryOperator(func, output) : (F) adaptLongUnaryOperatorStrict(func, output); + return truncate ? (F) adaptLongUnaryOperator(func, output) : + (F) adaptLongUnaryOperatorStrict(func, output); case DoubleUnaryOperator: - return truncate ? (F) adaptDoubleUnaryOperator(func, output) : (F) adaptDoubleUnaryOperatorStrict(func, output); + return truncate ? (F) adaptDoubleUnaryOperator(func, output) : + (F) adaptDoubleUnaryOperatorStrict(func, output); case IntUnaryOperator: - return truncate ? adaptIntUnaryOperator(func, output) : adaptIntUnaryOperatorStrict(func, output); + return truncate ? adaptIntUnaryOperator(func, output) : + adaptIntUnaryOperatorStrict(func, output); case DoubleFunction: - return truncate ? adaptDoubleFunction(func, output) : adaptDoubleFunctionStrict(func, output); + return truncate ? adaptDoubleFunction(func, output) : + adaptDoubleFunctionStrict(func, output); case LongFunction: - return truncate ? (F) adaptLongFunction(func, output) : (F) adaptLongFunctionStrict(func, output); + return truncate ? (F) adaptLongFunction(func, output) : + (F) adaptLongFunctionStrict(func, output); case LongToDoubleFunction: - return truncate ? (F) adaptLongToDoubleFunction(func, output) : (F) adaptLongToDoubleFunctionStrict(func, output); + return truncate ? (F) adaptLongToDoubleFunction(func, output) : + (F) adaptLongToDoubleFunctionStrict(func, output); case LongToIntFunction: - return truncate ? (F) adaptLongToIntFunction(func, output) : (F) adaptLongFunctionStrict(func, output); + return truncate ? (F) adaptLongToIntFunction(func, output) : + (F) adaptLongFunctionStrict(func, output); case IntFunction: return adaptIntFunction(func, output); case Function: - return truncate ? (F) adaptFunction(func, output) : adaptFunctionStrict(func, output); + return truncate ? (F) adaptFunction(func, output) : + adaptFunctionStrict(func, output); default: - throw new RuntimeException("Unable to convert function type '" + funcType + "' (" + func.getClass().getName() + - ") to " + type.getName() + (truncate ? " WITH " : " WITHOUT ") + "truncation"); + throw new RuntimeException( + "Unable to convert function type '" + funcType + "' (" + func.getClass() + .getName() + ") to " + type.getName() + (truncate ? " WITH " : " WITHOUT ") + + "truncation"); } } - public static List adaptList(Object[] funcs, Class type, Class output, boolean truncate) { + public static List adaptList( + Object[] funcs, + Class type, + Class output, + boolean truncate + ) + { List adapted = new ArrayList<>(); for (Object func : funcs) { F f = adapt(func, type, output, truncate); @@ -109,7 +134,11 @@ public class VirtDataFunctions { } - private static LongToDoubleFunction adaptLongToDoubleFunctionStrict(Object func, Class output) { + private static LongToDoubleFunction adaptLongToDoubleFunctionStrict( + Object func, + Class output + ) + { FuncType isaType = FuncType.valueOf(func.getClass()); switch (isaType) { case LongToDoubleFunction: @@ -119,62 +148,117 @@ public class VirtDataFunctions { LongToIntFunction f2 = assertTypesAssignable(func, LongToIntFunction.class); return f2::applyAsInt; case LongFunction: - LongFunction f3 = assertTypesAssignable(func, LongFunction.class, double.class); + LongFunction f3 = + assertTypesAssignable(func, LongFunction.class, double.class); return f3::apply; case LongUnaryOperator: LongUnaryOperator f4 = assertTypesAssignable(func, LongUnaryOperator.class); return f4::applyAsLong; case DoubleFunction: - DoubleFunction f7 = assertTypesAssignable(func, DoubleFunction.class, double.class); + DoubleFunction f7 = + assertTypesAssignable(func, DoubleFunction.class, double.class); return f7::apply; case DoubleUnaryOperator: DoubleUnaryOperator f8 = assertTypesAssignable(func, DoubleUnaryOperator.class); return f8::applyAsDouble; case Function: - Function f9 = assertTypesAssignable(func, Function.class, double.class, double.class); + Function f9 = + assertTypesAssignable(func, Function.class, double.class, double.class); return l -> f9.apply((double) l).doubleValue(); case IntFunction: case IntUnaryOperator: throwNarrowingError(func, isaType.functionClazz); default: - throw new BasicError("I don't know how to convert a " + func.getClass().getName() + " function to a LongToDoubleFunction."); + throw new BasicError("I don't know how to convert a " + func.getClass().getName() + + " function to a LongToDoubleFunction."); } } + private static DoubleToLongFunction adaptDoubleToLongFunction(Object func, Class output) { + FuncType isaType = FuncType.valueOf(func.getClass()); + switch (isaType) { + case LongToDoubleFunction: + LongToDoubleFunction f1 = assertTypesAssignable(func, LongToDoubleFunction.class); + return l -> (long) f1.applyAsDouble((long) l); + case LongToIntFunction: + LongToIntFunction f2 = assertTypesAssignable(func, LongToIntFunction.class); + return l -> f2.applyAsInt((long) l); + case LongFunction: + LongFunction f3 = assertTypesAssignable(func, LongFunction.class, Long.class); + return l -> f3.apply((long) l); + case LongUnaryOperator: + LongUnaryOperator f4 = assertTypesAssignable(func, LongUnaryOperator.class); + return l -> f4.applyAsLong((long) l); + case IntFunction: + IntFunction f5 = assertTypesAssignable(func, IntFunction.class, Long.class); + return l -> f5.apply((int) l); + case IntUnaryOperator: + IntUnaryOperator f6 = assertTypesAssignable(func, IntUnaryOperator.class); + return l -> f6.applyAsInt((int) l); + case DoubleToLongFunction: + DoubleToLongFunction f7 = assertTypesAssignable(func, DoubleToLongFunction.class); + return l -> f7.applyAsLong((long) l); + case DoubleToIntFunction: + DoubleToIntFunction f8 = assertTypesAssignable(func, DoubleToIntFunction.class); + return l -> f8.applyAsInt((int) l); + case DoubleFunction: + DoubleFunction f9 = + assertTypesAssignable(func, DoubleFunction.class, Long.class); + return f9::apply; + case DoubleUnaryOperator: + DoubleUnaryOperator f10 = assertTypesAssignable(func, DoubleUnaryOperator.class); + return l -> (long) f10.applyAsDouble(l); + case Function: + Function f11 = + assertTypesAssignable(func, Function.class, Double.class, Long.class); + return f11::apply; + default: + throw new BasicError("I don't know how to convert a " + func.getClass().getName() + + " function to a DoubleToLongFunction."); + } + + + } + + private static LongToDoubleFunction adaptLongToDoubleFunction(Object func, Class output) { FuncType isaType = FuncType.valueOf(func.getClass()); switch (isaType) { case LongToDoubleFunction: - LongToDoubleFunction f1 = assertTypesAssignable(func, LongToDoubleFunction.class); - return null; + return assertTypesAssignable(func, LongToDoubleFunction.class); case LongToIntFunction: LongToIntFunction f2 = assertTypesAssignable(func, LongToIntFunction.class); - return null; + return f2::applyAsInt; case LongFunction: - LongFunction f3 = assertTypesAssignable(func, LongFunction.class, double.class); - return null; + LongFunction f3 = + assertTypesAssignable(func, LongFunction.class, double.class); + return f3::apply; case LongUnaryOperator: LongUnaryOperator f4 = assertTypesAssignable(func, LongUnaryOperator.class); - return null; + return f4::applyAsLong; case IntFunction: - IntFunction f5 = assertTypesAssignable(func, IntFunction.class, double.class); - return null; + IntFunction f5 = + assertTypesAssignable(func, IntFunction.class, double.class); + return l -> f5.apply((int) l); case IntUnaryOperator: IntUnaryOperator f6 = assertTypesAssignable(func, IntUnaryOperator.class); - return null; + return operand -> f6.applyAsInt((int) operand); case DoubleFunction: - DoubleFunction f7 = assertTypesAssignable(func, DoubleFunction.class, double.class); - return null; + DoubleFunction f7 = + assertTypesAssignable(func, DoubleFunction.class, double.class); + return f7::apply; case DoubleUnaryOperator: DoubleUnaryOperator f8 = assertTypesAssignable(func, DoubleUnaryOperator.class); - return null; + return f8::applyAsDouble; case Function: - Function f9 = assertTypesAssignable(func, Function.class, double.class, double.class); - return null; + Function f9 = + assertTypesAssignable(func, Function.class, double.class, double.class); + return t -> f9.apply((double) t); default: - throw new BasicError("I don't know how to convert a " + func.getClass().getName() + " function to a LongToDoubleFunction."); + throw new BasicError("I don't know how to convert a " + func.getClass().getName() + + " function to a LongToDoubleFunction."); } } @@ -188,15 +272,32 @@ public class VirtDataFunctions { case LongUnaryOperator: LongUnaryOperator f2 = assertTypesAssignable(func, LongUnaryOperator.class); return f2::applyAsLong; + case LongToDoubleFunction: + LongToDoubleFunction f7 = assertTypesAssignable(func, LongToDoubleFunction.class); + return f7::applyAsDouble; case LongToIntFunction: LongToIntFunction f3 = assertTypesAssignable(func, LongToIntFunction.class); return f3::applyAsInt; + case DoubleToLongFunction: + DoubleToLongFunction f4 = assertTypesAssignable(func, DoubleToLongFunction.class); + return f4::applyAsLong; + case DoubleToIntFunction: + DoubleToIntFunction f5 = assertTypesAssignable(func, DoubleToIntFunction.class); + return f5::applyAsInt; + case DoubleFunction: + DoubleFunction f6 = assertTypesAssignable(func, DoubleFunction.class); + return f6::apply; case Function: - Function f7 = assertTypesAssignable(func, Function.class, Long.class); - return (long l) -> f7.apply(l); + Function f8 = assertTypesAssignable(func, Function.class, Long.class); + return f8::apply; + case DoubleUnaryOperator: + case IntUnaryOperator: + case IntFunction: default: - throw new RuntimeException("Unable to convert " + func.getClass().getName() + " to a " + - LongUnaryOperator.class.getName() + " since this would cause a narrowing conversion."); + throw new RuntimeException( + "Unable to convert " + func.getClass().getName() + " to a " + + LongUnaryOperator.class.getName() + + " since this would cause a narrowing conversion."); } } @@ -206,48 +307,60 @@ public class VirtDataFunctions { switch (isaType) { case LongFunction: LongFunction f1 = (LongFunction) func; - Function rf1 = f1::apply; - return rf1; + return (Function) f1::apply; case LongUnaryOperator: LongUnaryOperator f2 = (LongUnaryOperator) func; - Function rf2 = f2::applyAsLong; - return rf2; + return (Function) f2::applyAsLong; case IntFunction: - IntFunction f3 = (IntFunction) func; - Function rf3 = f3::apply; - return rf3; + IntFunction f3 = (IntFunction) func; + return (Function) f3::apply; case IntUnaryOperator: IntUnaryOperator f4 = (IntUnaryOperator) func; - Function rf4 = f4::applyAsInt; - return rf4; + return (Function) f4::applyAsInt; case DoubleFunction: - DoubleFunction f5 = (DoubleFunction) func; - Function rf5 = f5::apply; - return rf5; + DoubleFunction f5 = (DoubleFunction) func; + return (Function) f5::apply; case DoubleUnaryOperator: DoubleUnaryOperator f6 = (DoubleUnaryOperator) func; - Function rf6 = f6::applyAsDouble; - return rf6; + return (Function) f6::applyAsDouble; + case LongToDoubleFunction: + LongToDoubleFunction f10 = (LongToDoubleFunction) func; + return (Function) f10; case LongToIntFunction: LongToIntFunction f7 = (LongToIntFunction) func; Function rf7 = f7::applyAsInt; case Function: return (Function) func; + case DoubleToLongFunction: + DoubleToLongFunction f8 = (DoubleToLongFunction) func; + return (Function) f8::applyAsLong; + case DoubleToIntFunction: + DoubleToIntFunction f9 = (DoubleToIntFunction) func; + return (Function) f9; default: throw new RuntimeException("Unable to map function:" + func); } } private static F adaptFunctionStrict(Object func, Class output) { - throw new RuntimeException("This must be implemented, now that it is used: adaptFunctionsStrict(Object,Class)"); + throw new RuntimeException( + "This must be implemented, now that it is used: adaptFunctionsStrict(Object,Class)"); } private static F adaptDoubleFunctionStrict(Object func, Class output) { - throw new RuntimeException("This must be implemented, now that it is used: adaptDoubleFunctionStrict(Object,Class) "); + throw new RuntimeException( + "This must be implemented, now that it is used: adaptDoubleFunctionStrict(Object,Class) "); } + private static LongFunction adaptDoubleToLongFunctionStrict(Object func, Class output) { + throw new RuntimeException( + "This must be implemented, now that it is used: adaptDoubleToLongFunctionStrict(Object,Class)"); + } + + private static F adaptIntUnaryOperatorStrict(Object func, Class output) { - throw new RuntimeException("This must be implemented, now that it is used: adaptIntUnaryOperatorStrict(Object,Class)"); + throw new RuntimeException( + "This must be implemented, now that it is used: adaptIntUnaryOperatorStrict(Object,Class)"); } private static DoubleUnaryOperator adaptDoubleUnaryOperator(Object func, Class output) { @@ -255,7 +368,8 @@ public class VirtDataFunctions { switch (toFuncType) { case DoubleFunction: - DoubleFunction f2 = assertTypesAssignable(func, DoubleFunction.class, double.class); + DoubleFunction f2 = + assertTypesAssignable(func, DoubleFunction.class, double.class); return f2::apply; case LongToDoubleFunction: LongToDoubleFunction f3 = assertTypesAssignable(func, LongToDoubleFunction.class); @@ -264,13 +378,15 @@ public class VirtDataFunctions { LongToIntFunction f4 = assertTypesAssignable(func, LongToIntFunction.class); return l -> f4.applyAsInt((long) l % Long.MAX_VALUE); case LongFunction: - LongFunction f5 = assertTypesAssignable(func, LongFunction.class, double.class); + LongFunction f5 = + assertTypesAssignable(func, LongFunction.class, double.class); return l -> (double) f5.apply((long) l % Long.MAX_VALUE); case LongUnaryOperator: LongUnaryOperator f6 = assertTypesAssignable(func, LongUnaryOperator.class); return l -> f6.applyAsLong((long) l % Long.MAX_VALUE); case IntFunction: - IntFunction f7 = assertTypesAssignable(func, IntFunction.class, double.class); + IntFunction f7 = + assertTypesAssignable(func, IntFunction.class, double.class); return l -> (double) f7.apply((int) l % Integer.MAX_VALUE); case IntUnaryOperator: IntUnaryOperator f8 = assertTypesAssignable(func, IntUnaryOperator.class); @@ -282,25 +398,31 @@ public class VirtDataFunctions { DoubleUnaryOperator fA = assertTypesAssignable(func, DoubleUnaryOperator.class); return fA; case Function: - Function f1 = assertTypesAssignable(func, Function.class, double.class, double.class); + Function f1 = + assertTypesAssignable(func, Function.class, double.class, double.class); return f1::apply; default: throw new IllegalStateException("Unexpected value: " + toFuncType); } } - private static DoubleUnaryOperator adaptDoubleUnaryOperatorStrict(Object func, Class output) { + private static DoubleUnaryOperator adaptDoubleUnaryOperatorStrict(Object func, Class output) + { FuncType toFuncType = FuncType.valueOf(func.getClass()); switch (toFuncType) { case Function: - Function f1 = assertTypesAssignable(func, Function.class, double.class, double.class); + Function f1 = + assertTypesAssignable(func, Function.class, double.class, double.class); return f1::apply; case DoubleFunction: - DoubleFunction f2 = assertTypesAssignable(func, DoubleFunction.class, double.class); + DoubleFunction f2 = + assertTypesAssignable(func, DoubleFunction.class, double.class); return f2::apply; default: - throw new RuntimeException("Unable to convert " + func.getClass().getName() + " to a " + - DoubleUnaryOperator.class.getName() + " since this would cause a narrowing conversion."); + throw new RuntimeException( + "Unable to convert " + func.getClass().getName() + " to a " + + DoubleUnaryOperator.class.getName() + + " since this would cause a narrowing conversion."); } } @@ -315,16 +437,20 @@ public class VirtDataFunctions { LongUnaryOperator o5 = assertTypesAssignable(func, LongUnaryOperator.class); return o5; case Function: - Function o7 = assertTypesAssignable(func, Function.class, long.class, long.class); + Function o7 = + assertTypesAssignable(func, Function.class, long.class, long.class); return o7::apply; default: - throw new RuntimeException("Unable to convert " + func.getClass().getName() + " to a " + - LongUnaryOperator.class.getName() + " since this would cause a narrowing conversion."); + throw new RuntimeException( + "Unable to convert " + func.getClass().getName() + " to a " + + LongUnaryOperator.class.getName() + + " since this would cause a narrowing conversion."); } } private static F adaptIntFunction(Object func, Class output) { - throw new RuntimeException("This must be implemented, now that it is used: adaptIntFunction(Object,Class)"); + throw new RuntimeException( + "This must be implemented, now that it is used: adaptIntFunction(Object,Class)"); } protected static LongToIntFunction adaptLongToIntFunction(Object func, Class output) { @@ -332,31 +458,36 @@ public class VirtDataFunctions { switch (isaType) { case LongToDoubleFunction: - LongToDoubleFunction f1 = assertTypesAssignable(func, LongToDoubleFunction.class, double.class); + LongToDoubleFunction f1 = + assertTypesAssignable(func, LongToDoubleFunction.class, double.class); return l -> (int) (f1.applyAsDouble(l) % Integer.MAX_VALUE); case LongToIntFunction: LongToIntFunction f2 = assertTypesAssignable(func, LongToIntFunction.class); return f2; case LongFunction: - LongFunction f3 = assertTypesAssignable(func, LongFunction.class, double.class); + LongFunction f3 = + assertTypesAssignable(func, LongFunction.class, double.class); return l -> (int) f3.apply((int) l % Integer.MAX_VALUE).longValue(); case LongUnaryOperator: LongUnaryOperator f4 = assertTypesAssignable(func, LongUnaryOperator.class); return l -> (int) (f4.applyAsLong(l) % Integer.MAX_VALUE); case IntFunction: IntFunction f5 = assertTypesAssignable(func, IntFunction.class, double.class); - return l -> (int) f5.apply((int) l % Integer.MAX_VALUE).longValue() % Integer.MAX_VALUE; + return l -> (int) f5.apply((int) l % Integer.MAX_VALUE).longValue() + % Integer.MAX_VALUE; case IntUnaryOperator: IntUnaryOperator f6 = assertTypesAssignable(func, IntUnaryOperator.class); return l -> f6.applyAsInt((int) l % Integer.MAX_VALUE); case DoubleFunction: - DoubleFunction f7 = assertTypesAssignable(func, DoubleFunction.class, double.class); + DoubleFunction f7 = + assertTypesAssignable(func, DoubleFunction.class, double.class); return l -> (int) f7.apply(l).longValue() & Integer.MAX_VALUE; case DoubleUnaryOperator: DoubleUnaryOperator f8 = assertTypesAssignable(func, DoubleUnaryOperator.class); return l -> (int) f8.applyAsDouble(l) % Integer.MAX_VALUE; case Function: - Function f9 = assertTypesAssignable(func, Function.class, double.class, double.class); + Function f9 = + assertTypesAssignable(func, Function.class, double.class, double.class); return l -> (int) f9.apply((double) l).longValue() % Integer.MAX_VALUE; default: throw new IllegalStateException("Unexpected value: " + isaType); @@ -397,29 +528,33 @@ public class VirtDataFunctions { assertOutputAssignable(f9.applyAsInt(1L), output); return (long l) -> f9.applyAsInt(l); default: - throw new RuntimeException("Unable to convert " + func.getClass().getName() + " to a " + - LongUnaryOperator.class.getName()); + throw new RuntimeException( + "Unable to convert " + func.getClass().getName() + " to a " + + LongUnaryOperator.class.getName()); } } private static void assertOutputAssignable(Object result, Class clazz) { if (!ClassUtils.isAssignable(result.getClass(), clazz, true)) { - throw new InvalidParameterException("Unable to assign type of " + result.getClass().getName() - + " to " + clazz.getName()); + throw new InvalidParameterException( + "Unable to assign type of " + result.getClass().getName() + " to " + + clazz.getName()); } -// if (!clazz.isAssignableFrom(result.getClass())) { -// throw new InvalidParameterException("Unable to assign type of " + result.getClass().getName() -// + " to " + clazz.getName()); -// } + // if (!clazz.isAssignableFrom(result.getClass())) { + // throw new InvalidParameterException("Unable to assign type of " + result.getClass().getName() + // + " to " + clazz.getName()); + // } } private static F adaptDoubleFunction(Object func, Class output) { - throw new RuntimeException("This must be implemented, now that it is used: adaptDoubleFunction(Object,Class)"); + throw new RuntimeException( + "This must be implemented, now that it is used: adaptDoubleFunction(Object,Class)"); } private static F adaptIntUnaryOperator(Object func, Class output) { - throw new RuntimeException("This must be implemented, now that it is used: adaptIntUnaryOperator(Object,Class)"); + throw new RuntimeException( + "This must be implemented, now that it is used: adaptIntUnaryOperator(Object,Class)"); } @@ -434,7 +569,8 @@ public class VirtDataFunctions { LongFunction o2 = assertTypesAssignable(func, LongFunction.class, long.class); return o2::apply; case DoubleFunction: - DoubleFunction o3 = assertTypesAssignable(func, DoubleFunction.class, long.class); + DoubleFunction o3 = + assertTypesAssignable(func, DoubleFunction.class, long.class); return o3::apply; case IntUnaryOperator: IntUnaryOperator o4 = assertTypesAssignable(func, IntUnaryOperator.class); @@ -446,7 +582,8 @@ public class VirtDataFunctions { DoubleUnaryOperator o6 = assertTypesAssignable(func, DoubleUnaryOperator.class); return (long l) -> (long) (o6.applyAsDouble(l) % Long.MAX_VALUE); case Function: - Function o7 = assertTypesAssignable(func, Function.class, long.class, long.class); + Function o7 = + assertTypesAssignable(func, Function.class, long.class, long.class); return o7::apply; case LongToDoubleFunction: LongToDoubleFunction o8 = assertTypesAssignable(func, LongToDoubleFunction.class); @@ -455,49 +592,60 @@ public class VirtDataFunctions { LongToIntFunction o9 = assertTypesAssignable(func, LongToIntFunction.class); return o9::applyAsInt; } - throw new InvalidParameterException("Unable to convert " + func.getClass().getName() + " to a " + - LongUnaryOperator.class.getName()); + throw new InvalidParameterException( + "Unable to convert " + func.getClass().getName() + " to a " + + LongUnaryOperator.class.getName()); } /** - * Given a base object and a wanted type to convert it to, assert that the type of the base object is assignable to - * the wanted type. Further, if the wanted type is a generic type, assert that additional classes are assignable to - * the generic type parameters. Thus, if you want to assign to a generic type from a non-generic type, you must - * qualify the types of values that will be used in those generic parameter positions in declaration order. - * - *

This is useful for taking any object and a known type and reifying it as the known type so that it can be - * then used idiomatically with normal type awareness. This scenario occurs when you are accepting an open type for - * flexibility but then need to narrow the type sufficiently for additional conversion in a type-safe way.

- * - * @param base The object to be assigned to the wanted type - * @param wantType The class type that the base object needs to be assignable to - * @param clazzes The types of values which will checked against generic type parameters of the wanted type - * @param Generic parameter T for the wanted type - * @return The original object casted to the wanted type after verification of parameter assignability + Given a base object and a wanted type to convert it to, assert that the type of the base object + is assignable to + the wanted type. Further, if the wanted type is a generic type, assert that additional classes + are assignable to + the generic type parameters. Thus, if you want to assign to a generic type from a non-generic + type, you must + qualify the types of values that will be used in those generic parameter positions in + declaration order. + +

This is useful for taking any object and a known type and reifying it as the known type so + that it can be + then used idiomatically with normal type awareness. This scenario occurs when you are accepting + an open type for + flexibility but then need to narrow the type sufficiently for additional conversion in a + type-safe way.

+ @param base + The object to be assigned to the wanted type + @param wantType + The class type that the base object needs to be assignable to + @param clazzes + The types of values which will checked against generic type parameters of the wanted type + @param + Generic parameter T for the wanted type + @return The original object casted to the wanted type after verification of parameter assignability */ - private static T assertTypesAssignable( - Object base, - Class wantType, - Class... clazzes) { + private static T assertTypesAssignable(Object base, Class wantType, Class... clazzes) + { if (!wantType.isAssignableFrom(base.getClass())) { - throw new InvalidParameterException("Unable to assign " + wantType.getName() + " from " + - base.getClass().getName()); + throw new InvalidParameterException( + "Unable to assign " + wantType.getName() + " from " + base.getClass().getName()); } TypeVariable>[] typeParameters = base.getClass().getTypeParameters(); if (typeParameters.length > 0) { if (clazzes.length != typeParameters.length) { throw new InvalidParameterException( - "type parameter lengths are mismatched:" + clazzes.length + ", " + typeParameters.length - ); + "type parameter lengths are mismatched:" + clazzes.length + ", " + + typeParameters.length); } for (int i = 0; i < clazzes.length; i++) { Class from = clazzes[i]; TypeVariable> to = typeParameters[i]; boolean assignableFrom = to.getGenericDeclaration().isAssignableFrom(from); if (!assignableFrom) { - throw new InvalidParameterException("Can not assign " + from.getName() + " to " + to.getGenericDeclaration().getName()); + throw new InvalidParameterException( + "Can not assign " + from.getName() + " to " + to.getGenericDeclaration() + .getName()); } } @@ -507,14 +655,16 @@ public class VirtDataFunctions { } /** - * Throw an error indicating a narrowing conversion was attempted for strict conversion. - * - * @param func The source function to convert from - * @param targetClass The target class which was requested + Throw an error indicating a narrowing conversion was attempted for strict conversion. + @param func + The source function to convert from + @param targetClass + The target class which was requested */ private static void throwNarrowingError(Object func, Class targetClass) { - throw new BasicError("Converting from " + func.getClass().getName() + " to " + targetClass.getName() + - " is not allowed when strict conversion is requested."); + throw new BasicError( + "Converting from " + func.getClass().getName() + " to " + targetClass.getName() + + " is not allowed when strict conversion is requested."); } From cb4553eb40a266457029abe142aeeed20d5a6bf5 Mon Sep 17 00:00:00 2001 From: Jonathan Shook Date: Tue, 18 Feb 2025 17:56:38 -0600 Subject: [PATCH 5/6] more function in and out types --- .../to_long/IntervalHistribution.java | 78 +++++++++++ .../to_long/UnitHistribution.java} | 38 +++--- .../shared/from_long/to_double/HashMix.java | 6 +- .../shared/from_long/to_long/HashMix.java | 122 ++++++++++++++++++ .../from_long/to_double/HashMixTest.java | 53 ++++++-- ...ionTest.java => UnitHistributionTest.java} | 13 +- 6 files changed, 272 insertions(+), 38 deletions(-) create mode 100644 nb-virtdata/virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_double/to_long/IntervalHistribution.java rename nb-virtdata/virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/{from_long/to_double/EmpiricalHistribution.java => from_double/to_long/UnitHistribution.java} (67%) create mode 100644 nb-virtdata/virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_long/HashMix.java rename nb-virtdata/virtdata-lib-basics/src/test/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_double/{EmpiricalHistributionTest.java => UnitHistributionTest.java} (86%) diff --git a/nb-virtdata/virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_double/to_long/IntervalHistribution.java b/nb-virtdata/virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_double/to_long/IntervalHistribution.java new file mode 100644 index 000000000..6f34f8680 --- /dev/null +++ b/nb-virtdata/virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_double/to_long/IntervalHistribution.java @@ -0,0 +1,78 @@ +package io.nosqlbench.virtdata.library.basics.shared.from_double.to_long; + +/* + * Copyright (c) nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + +import io.nosqlbench.virtdata.api.annotations.Categories; +import io.nosqlbench.virtdata.api.annotations.Category; +import io.nosqlbench.virtdata.api.annotations.Example; +import io.nosqlbench.virtdata.api.annotations.ThreadSafeMapper; +import io.nosqlbench.virtdata.api.bindings.VirtDataFunctions; +import io.nosqlbench.virtdata.library.basics.core.stathelpers.AliasSamplerDoubleLong; +import io.nosqlbench.virtdata.library.basics.core.stathelpers.EvProbLongDouble; +import io.nosqlbench.virtdata.library.basics.shared.from_long.to_double.EmpiricalDistribution; +import io.nosqlbench.virtdata.library.basics.shared.from_long.to_double.HashRange; + +import java.util.ArrayList; +import java.util.List; +import java.util.function.DoubleToLongFunction; +import java.util.function.LongToDoubleFunction; +import java.util.function.LongUnaryOperator; + +@ThreadSafeMapper +@Categories(Category.distributions) +public class IntervalHistribution implements LongUnaryOperator { + + private final UnitHistribution sampler; + private final LongToDoubleFunction samplePointF; + + @Example({"IntervalHistribution('50 25 13 12')", "implied frequencies of 0:50 1:25 2:13 3:12"}) + @Example({ + "IntervalHistribution('234:50 33:25 17:13 3:12')", + "labeled frequencies; 234,33,17,3 are labels, and 50,25,13,12 are weights" + }) + public IntervalHistribution(String freqs, Object samplePointFunc) { + this.sampler = new UnitHistribution(freqs); + this.samplePointF = VirtDataFunctions.adapt( + samplePointFunc, + LongToDoubleFunction.class, + double.class, + false + ); + } + + public IntervalHistribution(String freqs) { + this(freqs,new HashRange(0.0d,1.0d)); + } + + + + private static List genEvents(long[] freqs) { + ArrayList events = new ArrayList<>(); + for (int i = 0; i < freqs.length; i++) { + events.add(new EvProbLongDouble(i, freqs[i])); + } + return events; + } + + @Override + public long applyAsLong(long operand) { + double samplePoint = this.samplePointF.applyAsDouble(operand); + return this.sampler.applyAsLong(samplePoint); + } +} diff --git a/nb-virtdata/virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_double/EmpiricalHistribution.java b/nb-virtdata/virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_double/to_long/UnitHistribution.java similarity index 67% rename from nb-virtdata/virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_double/EmpiricalHistribution.java rename to nb-virtdata/virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_double/to_long/UnitHistribution.java index ffee03a4e..8fe50ac5b 100644 --- a/nb-virtdata/virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_double/EmpiricalHistribution.java +++ b/nb-virtdata/virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_double/to_long/UnitHistribution.java @@ -1,4 +1,4 @@ -package io.nosqlbench.virtdata.library.basics.shared.from_long.to_double; +package io.nosqlbench.virtdata.library.basics.shared.from_double.to_long; /* * Copyright (c) nosqlbench @@ -18,16 +18,17 @@ package io.nosqlbench.virtdata.library.basics.shared.from_long.to_double; */ -import io.nosqlbench.nb.api.errors.BasicError; import io.nosqlbench.virtdata.api.annotations.Categories; import io.nosqlbench.virtdata.api.annotations.Category; import io.nosqlbench.virtdata.api.annotations.Example; import io.nosqlbench.virtdata.api.annotations.ThreadSafeMapper; import io.nosqlbench.virtdata.library.basics.core.stathelpers.AliasSamplerDoubleLong; import io.nosqlbench.virtdata.library.basics.core.stathelpers.EvProbLongDouble; +import io.nosqlbench.virtdata.library.basics.shared.from_long.to_double.EmpiricalDistribution; import java.util.ArrayList; import java.util.List; +import java.util.function.DoubleToLongFunction; /// Empirical Histribution is a portmanteau name to capture the /// concept of an empirical distribution based on a discrete histogram. @@ -38,14 +39,14 @@ import java.util.List; /// want to represent accurately. @ThreadSafeMapper @Categories(Category.distributions) -public class EmpiricalHistribution extends AliasSamplerDoubleLong { +public class UnitHistribution extends AliasSamplerDoubleLong implements DoubleToLongFunction { - @Example({"EmpiricalHistribution('50 25 13 12')", "implied frequencies of 0:50 1:25 2:13 3:12"}) + @Example({"UnitHistribution('50 25 13 12')", "implied frequencies of 0:50 1:25 2:13 3:12"}) @Example({ - "EmpiricalHistribution('234:50 33:25 17:13 3:12')", + "UnitHistribution('234:50 33:25 17:13 3:12')", "labeled frequencies; 234,33,17,3 are labels, and 50,25,13,12 are weights" }) - public EmpiricalHistribution(String freqs) { + public UnitHistribution(String freqs) { List events = new ArrayList<>(); boolean labeled = (freqs.contains(":")); @@ -57,20 +58,21 @@ public class EmpiricalHistribution extends AliasSamplerDoubleLong { "If any elements are labeled, all elements must be:" + freqs); } long id = labeled ? Long.parseLong(parts[0]) : i; - events.add(new EvProbLongDouble(id, Long.parseLong(parts[1]))); + long weight = Long.parseLong(parts[labeled ? 1 : 0]); + events.add(new EvProbLongDouble(id, weight)); } super(events); } - public EmpiricalHistribution(long... freqs) { - super(genEvents(freqs)); - } - - private static List genEvents(long[] freqs) { - ArrayList events = new ArrayList<>(); - for (int i = 0; i < freqs.length; i++) { - events.add(new EvProbLongDouble(i, freqs[i])); - } - return events; - } + // public UnitHistribution(long... freqs) { + // super(genEvents(freqs)); + // } + // + // private static List genEvents(long[] freqs) { + // ArrayList events = new ArrayList<>(); + // for (int i = 0; i < freqs.length; i++) { + // events.add(new EvProbLongDouble(i, freqs[i])); + // } + // return events; + // } } diff --git a/nb-virtdata/virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_double/HashMix.java b/nb-virtdata/virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_double/HashMix.java index 6d15eeebe..71dd09ab7 100644 --- a/nb-virtdata/virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_double/HashMix.java +++ b/nb-virtdata/virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_double/HashMix.java @@ -2,13 +2,13 @@ package io.nosqlbench.virtdata.library.basics.shared.from_long.to_double; /* * Copyright (c) nosqlbench - * + * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY diff --git a/nb-virtdata/virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_long/HashMix.java b/nb-virtdata/virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_long/HashMix.java new file mode 100644 index 000000000..16bf1d642 --- /dev/null +++ b/nb-virtdata/virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_long/HashMix.java @@ -0,0 +1,122 @@ +package io.nosqlbench.virtdata.library.basics.shared.from_long.to_long; + +/* + * Copyright (c) nosqlbench + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + +import io.nosqlbench.virtdata.api.annotations.Categories; +import io.nosqlbench.virtdata.api.annotations.Category; +import io.nosqlbench.virtdata.api.annotations.Example; +import io.nosqlbench.virtdata.api.annotations.ThreadSafeMapper; + +import java.util.function.LongToDoubleFunction; +import java.util.function.LongUnaryOperator; + +/// Blends two functions with a domain of 0..Long.MAX_VALUE as the input interval, +/// and a double output. The output value is interpolated between the output value +/// of the two according to the mix function. When the mix function yields a value +/// of 0.0, then the mix is turned _fully counter-clockwise_., or fully on the first provided +/// function. When the value is 1.0, the mix is turned all the clockwise, or fully on the second +/// provided function. +/// +/// If there are only two inner functions provided to HashMix, then it will default to +/// sampling random mixes at a randomized sample point. In other words, the variates +/// provided will be somewhere between the two curves on the unit interval. This is a simple way +/// to sample between two curves by default. The yielded value will be greater than or equal to +/// the lower of the two values at any point, and less than or equal to the greater of either. +/// +/// If a third parameter is provided to control the mix, then the mix can be set directly as a +/// unit interval. (The dial goes from 0.0 to 1.0). Any double or float value here will suffice. +/// You can use this when you want to have a test parameter that slews between two modeled +/// shapes. You can alternately provide any other function which can be coerced to a LongToDouble +/// function as a dynamic mix control. IFF such a function is provided, it must also be responsible +/// for hashing the input value if pseudo-randomness is desired. +/// +/// If a fourth parameter is provided, the sample point can also be controlled. By default, the +/// values on the provided curves will be sampled pseudo-randomly. However, a fourth parameter +/// can override this just like the mix ratio. As well, if you provide a value or function +/// to control the sample point, you are also responsible for any hashing needed to sample across +/// the whole space of possible values. +/// +/// The flexibility of these two parameters provides a substantial amount of flexibility. You +/// can, for example: +/// +/// - sample variates between two curves +/// - sample variates at a selected morphing step between the curves +/// - sample variates between two curves on a subsection of the unit interval +/// - sample variates within a defined band gap of the two curves +@ThreadSafeMapper +@Categories(Category.functional) +public class HashMix implements LongUnaryOperator { + + private io.nosqlbench.virtdata.library.basics.shared.from_long.to_double.HashMix mixer; + + + @Example({ + "IntervalHashMix(Func1(),Func2())", + "yield samples between func1 and func2 values at some random random sample point x" + }) + @Example({ + "IntervalHashMix(Func1(),Func2(),0.25d)", + "yield samples which are 25% from the sample values for func1 and func2 at some random " + + "sample point x" + }) + @Example({ + "IntervalHashMix(Func1(),Func2(),HashRange(0.25d,0.75d)", + "yield samples between 25% and 75% from func1 to func2 values at some random sample point x" + }) + @Example({ + "IntervalHashMix(Func1(),Func2(),0.0d,ScaledDouble())", + "access Func1 values as if it were the only one provided. ScaledDouble adds no " + + "randomization the input value, but it does map it to the sample domain of 0.0d-0.1d." + }) + public HashMix(Object curve1F, Object curve2F, Object mixPointF, Object samplePointF) { + this.mixer = new io.nosqlbench.virtdata.library.basics.shared.from_long.to_double.HashMix(curve1F, curve2F, mixPointF, samplePointF); + } + + public HashMix(Object curve1F, Object curve2F, Object mixPointF) { + this( + curve1F, + curve2F, + mixPointF, + new HashRange(Long.MAX_VALUE) + ); + } + + public HashMix(Object curve1F, Object curve2F) { + this( + curve1F, + curve2F, + new io.nosqlbench.virtdata.library.basics.shared.from_long.to_double.HashRange(0.0d, 1.0d), + new HashRange(Long.MAX_VALUE) + ); + } + + public HashMix(LongToDoubleFunction f1, LongToDoubleFunction f2) { + this( + f1, + f2, + new io.nosqlbench.virtdata.library.basics.shared.from_long.to_double.HashRange(0.0d, 1.0d), + new HashRange(Long.MAX_VALUE) + ); + } + + @Override + public long applyAsLong(long value) { + return (long) mixer.applyAsDouble(value); + } +} diff --git a/nb-virtdata/virtdata-lib-basics/src/test/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_double/HashMixTest.java b/nb-virtdata/virtdata-lib-basics/src/test/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_double/HashMixTest.java index c9df86b4a..a197e0012 100644 --- a/nb-virtdata/virtdata-lib-basics/src/test/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_double/HashMixTest.java +++ b/nb-virtdata/virtdata-lib-basics/src/test/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_double/HashMixTest.java @@ -2,13 +2,13 @@ package io.nosqlbench.virtdata.library.basics.shared.from_long.to_double; /* * Copyright (c) nosqlbench - * + * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -22,6 +22,7 @@ import org.assertj.core.data.Offset; import org.junit.jupiter.api.Test; import java.util.function.LongToDoubleFunction; +import java.util.function.LongUnaryOperator; import static org.assertj.core.api.Assertions.assertThat; @@ -33,12 +34,16 @@ public class HashMixTest { @Test public void testLinearMix() { - HashMix um1 = new HashMix(TO_UNIT_INTERVAL, TO_UNIT_INTERVAL); - for (long i = 1; i < (Long.MAX_VALUE >> 1); i *= 2) { - assertThat(um1.applyAsDouble(i)).isEqualTo( - TO_UNIT_INTERVAL.applyAsDouble(i), - Offset.offset(0.0000001d) - ); + DoubleHolder dh = new DoubleHolder(); + LongHolder lh = new LongHolder(); + HashMix um1 = new HashMix(TO_UNIT_INTERVAL, TO_UNIT_INTERVAL,dh, lh); + for (long i = 0; i >= 0L; i += 1L << 58) { + double fraction = TO_UNIT_INTERVAL.applyAsDouble(i); + lh.setValue(i); + dh.setValue(fraction); + double actual = um1.applyAsDouble(i); + double expected = TO_UNIT_INTERVAL.applyAsDouble(i); + assertThat(actual).isEqualTo(expected, Offset.offset(0.0000001d)); } } @@ -46,11 +51,37 @@ public class HashMixTest { public void testCrossfadeMix() { LongToDoubleFunction rampdown1 = l -> 1.0d - TO_UNIT_INTERVAL.applyAsDouble(l); LongToDoubleFunction rampdown2 = l -> 2.0d - TO_UNIT_INTERVAL.applyAsDouble(l); - HashMix um1 = new HashMix(rampdown1,rampdown2); - for (long i = 1<<24; i <= Long.MAX_VALUE>>1; i<<=1) { + HashMix um1 = new HashMix(rampdown1, rampdown2); + for (long i = 0; i >= 0L; i += 1L << 58) { double value = um1.applyAsDouble(i); assertThat(um1.applyAsDouble(i)).isEqualTo(1.0d, Offset.offset(0.0000001d)); } } + private class DoubleHolder implements LongToDoubleFunction { + + private double value; + + @Override + public double applyAsDouble(long value) { + return this.value; + } + + public void setValue(double value) { + this.value = value; + } + } + + private class LongHolder implements LongUnaryOperator { + private long value; + + @Override + public long applyAsLong(long operand) { + return this.value; + } + + public void setValue(long value) { + this.value = value; + } + } } diff --git a/nb-virtdata/virtdata-lib-basics/src/test/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_double/EmpiricalHistributionTest.java b/nb-virtdata/virtdata-lib-basics/src/test/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_double/UnitHistributionTest.java similarity index 86% rename from nb-virtdata/virtdata-lib-basics/src/test/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_double/EmpiricalHistributionTest.java rename to nb-virtdata/virtdata-lib-basics/src/test/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_double/UnitHistributionTest.java index 90bad9d9c..d709ce5d9 100644 --- a/nb-virtdata/virtdata-lib-basics/src/test/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_double/EmpiricalHistributionTest.java +++ b/nb-virtdata/virtdata-lib-basics/src/test/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_double/UnitHistributionTest.java @@ -2,13 +2,13 @@ package io.nosqlbench.virtdata.library.basics.shared.from_long.to_double; /* * Copyright (c) nosqlbench - * + * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -18,6 +18,7 @@ package io.nosqlbench.virtdata.library.basics.shared.from_long.to_double; */ +import io.nosqlbench.virtdata.library.basics.shared.from_double.to_long.UnitHistribution; import org.assertj.core.data.Offset; import org.junit.jupiter.api.Test; @@ -26,17 +27,17 @@ import java.util.Arrays; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; -public class EmpiricalHistributionTest { +public class UnitHistributionTest { @Test public void testUniformSyntaxRequired() { - assertThatThrownBy(() -> new EmpiricalHistribution("1 2:2 3:3")).hasMessageContaining( + assertThatThrownBy(() -> new UnitHistribution("1 2:2 3:3")).hasMessageContaining( "all elements must be"); } @Test public void testBasicHistribution() { - EmpiricalHistribution h = new EmpiricalHistribution("1:1 2:2 3:3"); + UnitHistribution h = new UnitHistribution("1:1 2:2 3:3"); long[] counts = new long[10]; int total=1000000; HashRange hr = new HashRange(0.0d, 1.0d); From b2da44bd1079c563b538668fdbdcbe78858467e6 Mon Sep 17 00:00:00 2001 From: Jonathan Shook Date: Tue, 18 Feb 2025 21:50:02 -0600 Subject: [PATCH 6/6] ninja fix cql driver changes --- .../geometry/NormalizeCqlVector.java | 26 ++++++++++--------- .../geometry/NormalizeCqlVectorTest.java | 4 +-- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/nb-adapters/adapter-cqld4/src/main/java/io/nosqlbench/datamappers/functions/geometry/NormalizeCqlVector.java b/nb-adapters/adapter-cqld4/src/main/java/io/nosqlbench/datamappers/functions/geometry/NormalizeCqlVector.java index ab4c4b104..a947e6a33 100644 --- a/nb-adapters/adapter-cqld4/src/main/java/io/nosqlbench/datamappers/functions/geometry/NormalizeCqlVector.java +++ b/nb-adapters/adapter-cqld4/src/main/java/io/nosqlbench/datamappers/functions/geometry/NormalizeCqlVector.java @@ -28,33 +28,35 @@ import java.util.List; import java.util.function.Function; /** - * Normalize a vector in List form, calling the appropriate conversion function - * depending on the component (Class) type of the incoming List values. - */ + Normalize a vector in List form, calling the appropriate conversion function + depending on the component (Class) type of the incoming List values. */ @ThreadSafeMapper @Categories(Category.experimental) -public class NormalizeCqlVector implements Function { +public class NormalizeCqlVector implements Function, CqlVector> { private final NormalizeDoubleListVector ndv = new NormalizeDoubleListVector(); private final NormalizeFloatListVector nfv = new NormalizeFloatListVector(); @Override - public CqlVector apply(CqlVector cqlVector) { + public CqlVector apply(CqlVector cqlVector) { double[] vals = new double[cqlVector.size()]; - double accumulator= 0.0d; + double accumulator = 0.0d; for (int i = 0; i < vals.length; i++) { - vals[i]=cqlVector.get(i).doubleValue(); - accumulator+=vals[i]*vals[i]; + vals[i] = cqlVector.get(i).doubleValue(); + accumulator += vals[i] * vals[i]; } - double factor = 1.0d/Math.sqrt(Arrays.stream(vals).map(d -> d * d).sum()); + double factor = 1.0d / Math.sqrt(Arrays.stream(vals).map(d -> d * d).sum()); if (cqlVector.get(0) instanceof Float) { - List list = Arrays.stream(vals).mapToObj(d -> Float.valueOf((float) (d * factor))).toList(); + List list = + Arrays.stream(vals).mapToObj(d -> Float.valueOf((float) (d * factor))).toList(); return CqlVector.newInstance(list); } else if (cqlVector.get(0) instanceof Double) { - List list = Arrays.stream(vals).mapToObj(d -> Double.valueOf((float) (d * factor))).toList(); + List list = + Arrays.stream(vals).mapToObj(d -> Double.valueOf((float) (d * factor))).toList(); return CqlVector.newInstance(list); } else { - throw new RuntimeException(NormalizeCqlVector.class.getCanonicalName()+ " only supports Double and Float type"); + throw new RuntimeException(NormalizeCqlVector.class.getCanonicalName() + + " only supports Double and Float type"); } } } diff --git a/nb-adapters/adapter-cqld4/src/test/java/io/nosqlbench/datamappers/functions/geometry/NormalizeCqlVectorTest.java b/nb-adapters/adapter-cqld4/src/test/java/io/nosqlbench/datamappers/functions/geometry/NormalizeCqlVectorTest.java index 02a06e118..a612a3b4e 100644 --- a/nb-adapters/adapter-cqld4/src/test/java/io/nosqlbench/datamappers/functions/geometry/NormalizeCqlVectorTest.java +++ b/nb-adapters/adapter-cqld4/src/test/java/io/nosqlbench/datamappers/functions/geometry/NormalizeCqlVectorTest.java @@ -46,9 +46,9 @@ public class NormalizeCqlVectorTest { @Test public void normalizeCqlVectorDoubles() { - CqlVector square = CqlVector.newInstance(1.0d, 1.0d); + CqlVector square = CqlVector.newInstance(1.0d, 1.0d); NormalizeCqlVector nv = new NormalizeCqlVector(); - CqlVector normaled = nv.apply(square); + CqlVector normaled = nv.apply(square); assertThat(normaled.size()).isEqualTo(2); assertThat(normaled.get(0)).isInstanceOf(Double.class);