IntenseFinance/nosqlbench
3
0
Fork 0
mirror of https://github.com/nosqlbench/nosqlbench.git synced 2025-02-25 18:55:28 -06:00
Code Issues Packages Projects Releases Wiki Activity

more paths consolidation

Browse Source
This commit is contained in:
Jonathan Shook
2020-04-01 19:01:04 -05:00
parent 24a0807cb0
commit 7afb19daca
18 changed files with 252 additions and 212 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/core/lfsrs/MetaShift.java
View File

@@ -1,6 +1,6 @@
package io.nosqlbench.virtdata.library.basics.core.lfsrs;
import io.nosqlbench.nb.api.pathutil.VirtDataResources;
import io.nosqlbench.nb.api.pathutil.NBPaths;
import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.LogManager;
@@ -141,7 +141,7 @@ public class MetaShift {
}
int availableSize= Math.max(registerSize,4);
String maskFileName= String.valueOf(availableSize)+"."+"txt";
List<String> lines = VirtDataResources.readDataFileLines("lfsrmasks/" + maskFileName);
List<String> lines = NBPaths.readDataFileLines("lfsrmasks/" + maskFileName);
long[] longs = lines.stream().mapToLong(s -> Long.parseLong(s, 16)).toArray();
return longs;
}

4
virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/distributions/CSVFrequencySampler.java
View File

@@ -14,7 +14,7 @@ import io.nosqlbench.virtdata.api.annotations.ThreadSafeMapper;
import io.nosqlbench.virtdata.library.basics.core.stathelpers.AliasSamplerDoubleInt;
import io.nosqlbench.virtdata.library.basics.core.stathelpers.EvProbD;
import io.nosqlbench.virtdata.library.basics.shared.from_long.to_long.Hash;
import io.nosqlbench.nb.api.pathutil.VirtDataResources;
import io.nosqlbench.nb.api.pathutil.NBPaths;
import org.apache.commons.csv.CSVParser;
import org.apache.commons.csv.CSVRecord;
import org.apache.commons.math3.stat.Frequency;
@@ -72,7 +72,7 @@ public class CSVFrequencySampler implements LongFunction<String> {
Set<String> values = new HashSet<>();
List<EvProbD> frequencies = new ArrayList<>();
CSVParser csvdata = VirtDataResources.readFileCSV(filename);
CSVParser csvdata = NBPaths.readFileCSV(filename);
Frequency freq = new Frequency();
for (CSVRecord csvdatum : csvdata) {
String value = csvdatum.get(columnName);

4
virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/distributions/DelimFrequencySampler.java
View File

@@ -14,7 +14,7 @@ import io.nosqlbench.virtdata.api.annotations.ThreadSafeMapper;
import io.nosqlbench.virtdata.library.basics.core.stathelpers.AliasSamplerDoubleInt;
import io.nosqlbench.virtdata.library.basics.core.stathelpers.EvProbD;
import io.nosqlbench.virtdata.library.basics.shared.from_long.to_long.Hash;
import io.nosqlbench.nb.api.pathutil.VirtDataResources;
import io.nosqlbench.nb.api.pathutil.NBPaths;
import org.apache.commons.csv.CSVParser;
import org.apache.commons.csv.CSVRecord;
import org.apache.commons.math3.stat.Frequency;
@@ -75,7 +75,7 @@ public class DelimFrequencySampler implements LongFunction<String> {
Set<String> values = new HashSet<>();
List<EvProbD> frequencies = new ArrayList<>();
CSVParser csvdata = VirtDataResources.readDelimFile(filename, delimiter);
CSVParser csvdata = NBPaths.readDelimFile(filename, delimiter);
Frequency freq = new Frequency();
for (CSVRecord csvdatum : csvdata) {
String value = csvdatum.get(columnName);

4
virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/distributions/WeightedStringsFromCSV.java
View File

@@ -6,7 +6,7 @@ import io.nosqlbench.virtdata.api.annotations.ThreadSafeMapper;
import io.nosqlbench.virtdata.library.basics.core.stathelpers.AliasSamplerDoubleInt;
import io.nosqlbench.virtdata.library.basics.shared.from_long.to_long.Hash;
import io.nosqlbench.virtdata.library.basics.core.stathelpers.EvProbD;
import io.nosqlbench.nb.api.pathutil.VirtDataResources;
import io.nosqlbench.nb.api.pathutil.NBPaths;
import org.apache.commons.csv.CSVParser;
import org.apache.commons.csv.CSVRecord;
@@ -71,7 +71,7 @@ public class WeightedStringsFromCSV implements LongFunction<String> {
this.hash=new Hash();
}
for (String filename: filenames) {
CSVParser csvdata = VirtDataResources.readFileCSV(filename);
CSVParser csvdata = NBPaths.readFileCSV(filename);
for (CSVRecord csvdatum : csvdata) {
String value = csvdatum.get(valueColumn);
values.add(value);

6
virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_int/HashedLineToInt.java
View File

@@ -1,13 +1,15 @@
package io.nosqlbench.virtdata.library.basics.shared.from_long.to_int;
import io.nosqlbench.virtdata.api.annotations.ThreadSafeMapper;
import io.nosqlbench.nb.api.pathutil.VirtDataResources;
import io.nosqlbench.nb.api.pathutil.NBPaths;
import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.LogManager;
import java.util.List;
import java.util.function.LongToIntFunction;
import static io.nosqlbench.nb.api.pathutil.NBPaths.readDataFileLines;
/**
* Return a pseudo-randomly selected integer value from a file of numeric values.
* Each line in the file must contain one parsable integer value.
@@ -21,7 +23,7 @@ public class HashedLineToInt implements LongToIntFunction {
public HashedLineToInt(String filename) {
this.filename = filename;
List<String> lines = VirtDataResources.readDataFileLines(filename);
List<String> lines = readDataFileLines(filename);
this.values = lines.stream().mapToInt(Integer::parseInt).toArray();
this.intHash = new Hash();
}

4
virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_string/HashedFileExtractToString.java
View File

@@ -21,7 +21,7 @@ package io.nosqlbench.virtdata.library.basics.shared.from_long.to_string;
import io.nosqlbench.virtdata.api.annotations.Example;
import io.nosqlbench.virtdata.api.annotations.ThreadSafeMapper;
import io.nosqlbench.virtdata.library.basics.shared.from_long.to_int.HashRange;
import io.nosqlbench.nb.api.pathutil.VirtDataResources;
import io.nosqlbench.nb.api.pathutil.NBPaths;
import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.LogManager;
@@ -60,7 +60,7 @@ public class HashedFileExtractToString implements LongFunction<String> {
if (fileDataImage == null) {
synchronized (HashedFileExtractToString.class) {
if (fileDataImage == null) {
CharBuffer image= VirtDataResources.readDataFileToCharBuffer(fileName);
CharBuffer image= NBPaths.readDataFileToCharBuffer(fileName);
fileDataImage = image;
}
}

10
virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_string/HashedLineToString.java
View File

@@ -20,7 +20,7 @@ package io.nosqlbench.virtdata.library.basics.shared.from_long.to_string;
import io.nosqlbench.virtdata.api.annotations.ThreadSafeMapper;
import io.nosqlbench.virtdata.library.basics.shared.from_long.to_int.HashRange;
import io.nosqlbench.nb.api.pathutil.VirtDataResources;
import io.nosqlbench.nb.api.pathutil.NBPaths;
import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.LogManager;
@@ -28,13 +28,15 @@ import java.util.ArrayList;
import java.util.List;
import java.util.function.LongFunction;
import static io.nosqlbench.nb.api.pathutil.NBPaths.*;
/**
* Return a pseudo-randomly selected String value from a single line of
* the specified file.
*/
@ThreadSafeMapper
public class HashedLineToString implements LongFunction<String> {
private final static Logger logger = LogManager.getLogger(HashedLineToString.class);
private final static Logger logger = LogManager.getLogger(HashedLineToString.class);
private final HashRange indexRange;
private List<String> lines = new ArrayList<>();
@@ -43,8 +45,8 @@ public class HashedLineToString implements LongFunction<String> {
public HashedLineToString(String filename) {
this.filename = filename;
this.lines = VirtDataResources.readDataFileLines(filename);
this.indexRange = new HashRange(0, lines.size()-2);
this.lines = readDataFileLines(filename);
this.indexRange = new HashRange(0, lines.size() - 2);
}
public String toString() {

4
virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_string/ModuloCSVLineToString.java
View File

@@ -20,7 +20,7 @@ package io.nosqlbench.virtdata.library.basics.shared.from_long.to_string;
import io.nosqlbench.virtdata.api.annotations.Example;
import io.nosqlbench.virtdata.api.annotations.ThreadSafeMapper;
import io.nosqlbench.nb.api.pathutil.VirtDataResources;
import io.nosqlbench.nb.api.pathutil.NBPaths;
import org.apache.commons.csv.CSVParser;
import org.apache.commons.csv.CSVRecord;
import org.apache.logging.log4j.Logger;
@@ -47,7 +47,7 @@ public class ModuloCSVLineToString implements LongFunction<String> {
@Example({"ModuloCSVLineToString('data/myfile.csv','lat')","load values for 'lat' from the CSV file myfile.csv."})
public ModuloCSVLineToString(String filename, String fieldname) {
this.filename = filename;
CSVParser csvp = VirtDataResources.readFileCSV(filename);
CSVParser csvp = NBPaths.readFileCSV(filename);
Map<String, Integer> headerMap = csvp.getHeaderMap();
if (headerMap==null || headerMap.isEmpty()) {

6
virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_string/ModuloLineToString.java
View File

@@ -19,7 +19,7 @@
package io.nosqlbench.virtdata.library.basics.shared.from_long.to_string;
import io.nosqlbench.virtdata.api.annotations.ThreadSafeMapper;
import io.nosqlbench.nb.api.pathutil.VirtDataResources;
import io.nosqlbench.nb.api.pathutil.NBPaths;
import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.LogManager;
@@ -27,6 +27,8 @@ import java.util.ArrayList;
import java.util.List;
import java.util.function.LongFunction;
import static io.nosqlbench.nb.api.pathutil.NBPaths.readDataFileLines;
/**
* Select a value from a text file line by modulo division against the number
* of lines in the file.
@@ -40,7 +42,7 @@ public class ModuloLineToString implements LongFunction<String> {
public ModuloLineToString(String filename) {
this.filename = filename;
this.lines = VirtDataResources.readDataFileLines(filename);
this.lines = readDataFileLines(filename);
}
@Override