IntenseFinance/nosqlbench
3
0
Fork 0
mirror of https://github.com/nosqlbench/nosqlbench.git synced 2025-02-25 18:55:28 -06:00
Code Issues Packages Projects Releases Wiki Activity

partial rework of the NBIO internals for multiple results

Browse Source
This commit is contained in:
Jonathan Shook
2020-04-08 12:38:16 -05:00
parent d8b3e6214c
commit 1e05c31075
38 changed files with 240 additions and 673 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/core/lfsrs/MetaShift.java
View File

@@ -1,6 +1,6 @@
package io.nosqlbench.virtdata.library.basics.core.lfsrs;
import io.nosqlbench.nb.api.pathutil.NBPaths;
import io.nosqlbench.nb.api.content.NBIO;
import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.LogManager;
@@ -141,7 +141,7 @@ public class MetaShift {
}
int availableSize= Math.max(registerSize,4);
String maskFileName= String.valueOf(availableSize)+"."+"txt";
List<String> lines = NBPaths.readDataFileLines("lfsrmasks/" + maskFileName);
List<String> lines = NBIO.readLines("lfsrmasks/" + maskFileName);
long[] longs = lines.stream().mapToLong(s -> Long.parseLong(s, 16)).toArray();
return longs;
}

4
virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/distributions/CSVFrequencySampler.java
View File

@@ -7,6 +7,7 @@ package io.nosqlbench.virtdata.library.basics.shared.distributions;
*/
import io.nosqlbench.nb.api.content.NBIO;
import io.nosqlbench.virtdata.api.annotations.Categories;
import io.nosqlbench.virtdata.api.annotations.Category;
import io.nosqlbench.virtdata.api.annotations.Example;
@@ -14,7 +15,6 @@ import io.nosqlbench.virtdata.api.annotations.ThreadSafeMapper;
import io.nosqlbench.virtdata.library.basics.core.stathelpers.AliasSamplerDoubleInt;
import io.nosqlbench.virtdata.library.basics.core.stathelpers.EvProbD;
import io.nosqlbench.virtdata.library.basics.shared.from_long.to_long.Hash;
import io.nosqlbench.nb.api.pathutil.NBPaths;
import org.apache.commons.csv.CSVParser;
import org.apache.commons.csv.CSVRecord;
import org.apache.commons.math3.stat.Frequency;
@@ -72,7 +72,7 @@ public class CSVFrequencySampler implements LongFunction<String> {
Set<String> values = new HashSet<>();
List<EvProbD> frequencies = new ArrayList<>();
CSVParser csvdata = NBPaths.readFileCSV(filename);
CSVParser csvdata = NBIO.readFileCSV(filename);
Frequency freq = new Frequency();
for (CSVRecord csvdatum : csvdata) {
String value = csvdatum.get(columnName);

4
virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/distributions/DelimFrequencySampler.java
View File

@@ -7,6 +7,7 @@ package io.nosqlbench.virtdata.library.basics.shared.distributions;
*/
import io.nosqlbench.nb.api.content.NBIO;
import io.nosqlbench.virtdata.api.annotations.Categories;
import io.nosqlbench.virtdata.api.annotations.Category;
import io.nosqlbench.virtdata.api.annotations.Example;
@@ -14,7 +15,6 @@ import io.nosqlbench.virtdata.api.annotations.ThreadSafeMapper;
import io.nosqlbench.virtdata.library.basics.core.stathelpers.AliasSamplerDoubleInt;
import io.nosqlbench.virtdata.library.basics.core.stathelpers.EvProbD;
import io.nosqlbench.virtdata.library.basics.shared.from_long.to_long.Hash;
import io.nosqlbench.nb.api.pathutil.NBPaths;
import org.apache.commons.csv.CSVParser;
import org.apache.commons.csv.CSVRecord;
import org.apache.commons.math3.stat.Frequency;
@@ -75,7 +75,7 @@ public class DelimFrequencySampler implements LongFunction<String> {
Set<String> values = new HashSet<>();
List<EvProbD> frequencies = new ArrayList<>();
CSVParser csvdata = NBPaths.readDelimFile(filename, delimiter);
CSVParser csvdata = NBIO.readFileDelimCSV(filename,delimiter);
Frequency freq = new Frequency();
for (CSVRecord csvdatum : csvdata) {
String value = csvdatum.get(columnName);

4
virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/distributions/WeightedStringsFromCSV.java
View File

@@ -1,12 +1,12 @@
package io.nosqlbench.virtdata.library.basics.shared.distributions;
import io.nosqlbench.nb.api.content.NBIO;
import io.nosqlbench.virtdata.api.annotations.Categories;
import io.nosqlbench.virtdata.api.annotations.Category;
import io.nosqlbench.virtdata.api.annotations.ThreadSafeMapper;
import io.nosqlbench.virtdata.library.basics.core.stathelpers.AliasSamplerDoubleInt;
import io.nosqlbench.virtdata.library.basics.shared.from_long.to_long.Hash;
import io.nosqlbench.virtdata.library.basics.core.stathelpers.EvProbD;
import io.nosqlbench.nb.api.pathutil.NBPaths;
import org.apache.commons.csv.CSVParser;
import org.apache.commons.csv.CSVRecord;
@@ -71,7 +71,7 @@ public class WeightedStringsFromCSV implements LongFunction<String> {
this.hash=new Hash();
}
for (String filename: filenames) {
CSVParser csvdata = NBPaths.readFileCSV(filename);
CSVParser csvdata = NBIO.readFileCSV(filename);
for (CSVRecord csvdatum : csvdata) {
String value = csvdatum.get(valueColumn);
values.add(value);

5
virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_int/HashedLineToInt.java
View File

@@ -1,5 +1,6 @@
package io.nosqlbench.virtdata.library.basics.shared.from_long.to_int;
import io.nosqlbench.nb.api.content.NBIO;
import io.nosqlbench.virtdata.api.annotations.ThreadSafeMapper;
import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.LogManager;
@@ -7,8 +8,6 @@ import org.apache.logging.log4j.LogManager;
import java.util.List;
import java.util.function.LongToIntFunction;
import static io.nosqlbench.nb.api.pathutil.NBPaths.readDataFileLines;
/**
* Return a pseudo-randomly selected integer value from a file of numeric values.
* Each line in the file must contain one parsable integer value.
@@ -22,7 +21,7 @@ public class HashedLineToInt implements LongToIntFunction {
public HashedLineToInt(String filename) {
this.filename = filename;
List<String> lines = readDataFileLines(filename);
List<String> lines = NBIO.readLines(filename);
this.values = lines.stream().mapToInt(Integer::parseInt).toArray();
this.intHash = new Hash();
}

4
virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_string/HashedFileExtractToString.java
View File

@@ -18,10 +18,10 @@
package io.nosqlbench.virtdata.library.basics.shared.from_long.to_string;
import io.nosqlbench.nb.api.content.NBIO;
import io.nosqlbench.virtdata.api.annotations.Example;
import io.nosqlbench.virtdata.api.annotations.ThreadSafeMapper;
import io.nosqlbench.virtdata.library.basics.shared.from_long.to_int.HashRange;
import io.nosqlbench.nb.api.pathutil.NBPaths;
import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.LogManager;
@@ -60,7 +60,7 @@ public class HashedFileExtractToString implements LongFunction<String> {
if (fileDataImage == null) {
synchronized (HashedFileExtractToString.class) {
if (fileDataImage == null) {
CharBuffer image= NBPaths.readDataFileToCharBuffer(fileName);
CharBuffer image= NBIO.readCharBuffer(fileName);
fileDataImage = image;
}
}

5
virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_string/HashedLineToString.java
View File

@@ -18,6 +18,7 @@
package io.nosqlbench.virtdata.library.basics.shared.from_long.to_string;
import io.nosqlbench.nb.api.content.NBIO;
import io.nosqlbench.virtdata.api.annotations.ThreadSafeMapper;
import io.nosqlbench.virtdata.library.basics.shared.from_long.to_int.HashRange;
import org.apache.logging.log4j.Logger;
@@ -27,8 +28,6 @@ import java.util.ArrayList;
import java.util.List;
import java.util.function.LongFunction;
import static io.nosqlbench.nb.api.pathutil.NBPaths.*;
/**
* Return a pseudo-randomly selected String value from a single line of
* the specified file.
@@ -44,7 +43,7 @@ public class HashedLineToString implements LongFunction<String> {
public HashedLineToString(String filename) {
this.filename = filename;
this.lines = readDataFileLines(filename);
this.lines = NBIO.readLines(filename);
this.indexRange = new HashRange(0, lines.size() - 2);
}

4
virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_string/ModuloCSVLineToString.java
View File

@@ -18,9 +18,9 @@
package io.nosqlbench.virtdata.library.basics.shared.from_long.to_string;
import io.nosqlbench.nb.api.content.NBIO;
import io.nosqlbench.virtdata.api.annotations.Example;
import io.nosqlbench.virtdata.api.annotations.ThreadSafeMapper;
import io.nosqlbench.nb.api.pathutil.NBPaths;
import org.apache.commons.csv.CSVParser;
import org.apache.commons.csv.CSVRecord;
import org.apache.logging.log4j.Logger;
@@ -47,7 +47,7 @@ public class ModuloCSVLineToString implements LongFunction<String> {
@Example({"ModuloCSVLineToString('data/myfile.csv','lat')","load values for 'lat' from the CSV file myfile.csv."})
public ModuloCSVLineToString(String filename, String fieldname) {
this.filename = filename;
CSVParser csvp = NBPaths.readFileCSV(filename);
CSVParser csvp = NBIO.readFileCSV(filename);
Map<String, Integer> headerMap = csvp.getHeaderMap();
if (headerMap==null || headerMap.isEmpty()) {

6
virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_string/ModuloLineToString.java
View File

@@ -18,6 +18,7 @@
package io.nosqlbench.virtdata.library.basics.shared.from_long.to_string;
import io.nosqlbench.nb.api.content.NBIO;
import io.nosqlbench.virtdata.api.annotations.ThreadSafeMapper;
import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.LogManager;
@@ -26,8 +27,6 @@ import java.util.ArrayList;
import java.util.List;
import java.util.function.LongFunction;
import static io.nosqlbench.nb.api.pathutil.NBPaths.readDataFileLines;
/**
* Select a value from a text file line by modulo division against the number
* of lines in the file.
@@ -41,7 +40,8 @@ public class ModuloLineToString implements LongFunction<String> {
public ModuloLineToString(String filename) {
this.filename = filename;
this.lines = readDataFileLines(filename);
this.lines = NBIO.readLines(filename);
}
@Override