IntenseFinance/nosqlbench
3
0
Fork 0
mirror of https://github.com/nosqlbench/nosqlbench.git synced 2024-11-25 10:10:32 -06:00
Code Issues Packages Projects Releases Wiki Activity

add bindings for init-and-use byte buffers and char buffers

Browse Source
This commit is contained in:
Jonathan Shook 2021-04-16 15:37:58 -05:00
parent bf1a43fa41
commit 5f184e5592
5 changed files with 207 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_bytebuffer/ByteBufferSizedHashed.java
View File

@ -8,16 +8,16 @@ import io.nosqlbench.virtdata.api.bindings.VirtDataConversions;
import io.nosqlbench.virtdata.library.basics.shared.from_long.to_long.Hash;
import java.nio.ByteBuffer;
import java.util.HashMap;
import java.util.function.LongFunction;
import java.util.function.LongToIntFunction;
/**
* Create a ByteBuffer from a long input based on a provided size function.
*
* As a 'Sized' function, the first argument is a function which determines the size of the resulting ByteBuffer.
*
* As a 'Hashed' function, the input value is hashed again before being used as value.
*/
* Create a ByteBuffer from a long input based on a provided size function.
*
* As a 'Sized' function, the first argument is a function which determines the size of the resulting ByteBuffer.
*
* As a 'Hashed' function, the input value is hashed again before being used as value.
*/
@Categories(Category.conversion)
@ThreadSafeMapper
public class ByteBufferSizedHashed implements LongFunction<ByteBuffer> {
@ -27,28 +27,31 @@ public class ByteBufferSizedHashed implements LongFunction<ByteBuffer> {
@Example({
"ByteBufferSizedHashed(16)",
"Functionally identical to HashedtoByteBuffer(16) but using dynamic sizing implementation"
"ByteBufferSizedHashed(16)",
"Functionally identical to HashedtoByteBuffer(16) but using dynamic sizing implementation"
})
@Example({
"ByteBufferSizedHashed(HashRange(10, 14))",
"Create a ByteBuffer with variable limit (10 to 14)"
"ByteBufferSizedHashed(HashRange(10, 14))",
"Create a ByteBuffer with variable limit (10 to 14)"
})
public ByteBufferSizedHashed(int size) {
this.sizeFunc = s -> size;
}
public ByteBufferSizedHashed(Object sizeFunc) {
this.sizeFunc = VirtDataConversions.adaptFunction(sizeFunc, LongToIntFunction.class);
}
public ByteBufferSizedHashed(Object sizeFunc) {
if (sizeFunc instanceof Number) {
int size = ((Number) sizeFunc).intValue();
this.sizeFunc = s -> size;
} else {
this.sizeFunc = VirtDataConversions.adaptFunction(sizeFunc, LongToIntFunction.class);
}
}
@Override
public ByteBuffer apply(long input) {
int length = sizeFunc.applyAsInt(input);
int longs = (length / Long.BYTES) +1;
int longs = (length / Long.BYTES) + 1;
int bytes = longs * Long.BYTES;
ByteBuffer buffer = ByteBuffer.allocate(bytes);

18
virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_bytebuffer/ByteBufferToHex.java Normal file
View File

@ -0,0 +1,18 @@
package io.nosqlbench.virtdata.library.basics.shared.from_long.to_bytebuffer;
import io.nosqlbench.virtdata.api.annotations.ThreadSafeMapper;
import org.apache.commons.codec.binary.Hex;
import java.nio.ByteBuffer;
import java.util.function.Function;
/**
* Convert the contents of the input ByteBuffer to a String as hexadecimal.
*/
@ThreadSafeMapper
public class ByteBufferToHex implements Function<ByteBuffer,String> {
@Override
public String apply(ByteBuffer byteBuffer) {
return Hex.encodeHexString(byteBuffer);
}
}

58
virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_bytebuffer/HashedByteBufferExtract.java Normal file
View File

@ -0,0 +1,58 @@
package io.nosqlbench.virtdata.library.basics.shared.from_long.to_bytebuffer;
import io.nosqlbench.virtdata.api.bindings.VirtDataConversions;
import io.nosqlbench.virtdata.library.basics.shared.from_long.to_int.Hash;
import java.nio.ByteBuffer;
import java.util.function.LongFunction;
import java.util.function.LongToIntFunction;
/**
* Create a ByteBuffer from the first function, and then sample data from
* that bytebuffer according to the size function. The initFunction can be
* given as simply a size, in which case ByteBufferSizedHash is used.
* If the size function yields a size larger than the available buffer size, then it is
* lowered to that size automatically. If it is lower, then a random offset
* is used within the buffer image.
*
* This function behaves slightly differently than most in that it creates and
* caches as source byte buffer during initialization.
*/
public class HashedByteBufferExtract implements LongFunction<ByteBuffer> {
private final LongToIntFunction sizefunc;
private final ThreadLocal<ByteBuffer> bbaccessor;
private final Hash inthash = new Hash();
public HashedByteBufferExtract(Object initFunc, Object sizeFunc) {
ByteBuffer image = null;
if (initFunc instanceof Number) {
int bufsize = ((Number)initFunc).intValue();
ByteBufferSizedHashed bufgen = new ByteBufferSizedHashed(bufsize);
image = bufgen.apply(0).asReadOnlyBuffer();
} else {
LongFunction<ByteBuffer> bbfunc = VirtDataConversions.adaptFunction(initFunc, LongFunction.class, ByteBuffer.class);
image = bbfunc.apply(0).asReadOnlyBuffer();
}
ByteBuffer finalImage = image;
bbaccessor = ThreadLocal.withInitial(() -> finalImage.asReadOnlyBuffer());
if (sizeFunc instanceof Number) {
int size = ((Number)sizeFunc).intValue();
this.sizefunc = l -> size;
} else {
this.sizefunc = VirtDataConversions.adaptFunction(sizeFunc, LongToIntFunction.class);
}
}
@Override
public ByteBuffer apply(long value) {
ByteBuffer bbimage = bbaccessor.get();
int newbufsize = sizefunc.applyAsInt(value);
newbufsize=Math.min(newbufsize,bbimage.capacity());
byte[] bytes = new byte[newbufsize];
int base_offset = inthash.applyAsInt(value) % (bbimage.capacity()-bytes.length);
bbaccessor.get().position(base_offset).get(bytes);
return ByteBuffer.wrap(bytes);
}
}

62
virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_bytebuffer/HashedCharBufferExtract.java Normal file
View File

@ -0,0 +1,62 @@
package io.nosqlbench.virtdata.library.basics.shared.from_long.to_bytebuffer;
import io.nosqlbench.virtdata.api.bindings.VirtDataConversions;
import io.nosqlbench.virtdata.library.basics.shared.from_long.to_int.Hash;
import org.apache.commons.codec.binary.Hex;
import java.nio.ByteBuffer;
import java.nio.CharBuffer;
import java.util.function.LongFunction;
import java.util.function.LongToIntFunction;
/**
* Create a CharBuffer from the first function, and then sample data from
* that buffer according to the size function. The initFunction can be
* given as simply a size, in which case ByteBufferSizedHash is used with Hex String
* conversion.
* If the size function yields a size larger than the available buffer size, then it is
* lowered to that size automatically. If it is lower, then a random offset
* is used within the buffer image.
*
* This function behaves slightly differently than most in that it creates and
* caches as source byte buffer during initialization.
*/
public class HashedCharBufferExtract implements LongFunction<CharBuffer> {
private final LongToIntFunction sizefunc;
private final ThreadLocal<CharBuffer> bbaccessor;
private final Hash inthash = new Hash();
public HashedCharBufferExtract(Object initFunc, Object sizeFunc) {
CharBuffer image = null;
if (initFunc instanceof Number) {
int bufsize = ((Number)initFunc).intValue();
ByteBufferSizedHashed bufgen = new ByteBufferSizedHashed(bufsize);
ByteBuffer bbimage = bufgen.apply(0).asReadOnlyBuffer();
image = CharBuffer.wrap(Hex.encodeHex(bbimage));
} else {
LongFunction<String> bbfunc = VirtDataConversions.adaptFunction(initFunc, LongFunction.class, String.class);
image = CharBuffer.wrap(bbfunc.apply(0));
}
CharBuffer finalImage = image;
bbaccessor = ThreadLocal.withInitial(() -> finalImage.asReadOnlyBuffer());
if (sizeFunc instanceof Number) {
int size = ((Number)sizeFunc).intValue();
this.sizefunc = l -> size;
} else {
this.sizefunc = VirtDataConversions.adaptFunction(sizeFunc, LongToIntFunction.class);
}
}
@Override
public CharBuffer apply(long value) {
CharBuffer bbimage = bbaccessor.get();
int newbufsize = sizefunc.applyAsInt(value);
newbufsize=Math.min(newbufsize,bbimage.capacity());
char[] chars = new char[newbufsize];
int base_offset = inthash.applyAsInt(value) % (bbimage.capacity()-chars.length);
bbaccessor.get().position(base_offset).get(chars);
return CharBuffer.wrap(chars);
}
}

49
virtdata-lib-basics/src/test/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_bytebuffer/HashedByteBufferExtractTest.java Normal file
View File

@ -0,0 +1,49 @@
package io.nosqlbench.virtdata.library.basics.shared.from_long.to_bytebuffer;
import io.nosqlbench.virtdata.library.basics.shared.from_long.to_string.HashedLoremExtractToString;
import org.apache.commons.codec.binary.Hex;
import org.junit.Test;
import java.nio.ByteBuffer;
import java.nio.CharBuffer;
import java.util.function.LongToIntFunction;
public class HashedByteBufferExtractTest {
@Test
public void read1MBBytesDefault() {
HashedByteBufferExtract bbe = new HashedByteBufferExtract(1024*1024,(LongToIntFunction) l -> 10);
for (int i = 0; i < 10; i++) {
ByteBuffer a0 = bbe.apply(i);
System.out.println(Hex.encodeHex(a0));
}
}
@Test
public void read1MBBytesFunction() {
HashedByteBufferExtract bbe = new HashedByteBufferExtract(new ByteBufferSizedHashed(1024*1024),(LongToIntFunction) l -> 10);
for (int i = 0; i < 10; i++) {
ByteBuffer a0 = bbe.apply(i);
System.out.println(Hex.encodeHex(a0));
}
}
@Test
public void read1MBChars() {
HashedCharBufferExtract bbe = new HashedCharBufferExtract(1024*1024,(LongToIntFunction) l -> 10);
for (int i = 0; i < 10; i++) {
CharBuffer a0 = bbe.apply(i);
System.out.println(a0.toString());
}
}
@Test
public void read1MBCharsFunction() {
HashedCharBufferExtract bbe = new HashedCharBufferExtract(new HashedLoremExtractToString(1000,1000),(LongToIntFunction) l -> 10);
for (int i = 0; i < 10; i++) {
CharBuffer a0 = bbe.apply(i);
System.out.println(a0.toString());
}
}
}