IntenseFinance/nosqlbench
3
0
Fork 0
mirror of https://github.com/nosqlbench/nosqlbench.git synced 2025-02-25 18:55:28 -06:00
Code Issues Packages Projects Releases Wiki Activity

more paths consolidation

Browse Source
This commit is contained in:
Jonathan Shook
2020-04-01 19:01:04 -05:00
parent 24a0807cb0
commit 7afb19daca
18 changed files with 252 additions and 212 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
activitytype-cql/src/main/java/io/nosqlbench/activitytype/cql/datamappers/functions/long_string/ModuloCSVLineToUUID.java
View File

@@ -21,7 +21,7 @@ package io.nosqlbench.activitytype.cql.datamappers.functions.long_string;
import io.nosqlbench.virtdata.api.annotations.Example;
import io.nosqlbench.virtdata.api.annotations.ThreadSafeMapper;
import io.nosqlbench.nb.api.pathutil.VirtDataResources;
import io.nosqlbench.nb.api.pathutil.NBPaths;
import io.nosqlbench.virtdata.library.basics.shared.from_long.to_string.ModuloLineToString;
import org.apache.commons.csv.CSVParser;
import org.apache.commons.csv.CSVRecord;
@@ -49,7 +49,7 @@ public class ModuloCSVLineToUUID implements LongFunction<UUID> {
@Example({"ModuloCSVLineToUUID('data/myfile.csv','lat')","load values for 'lat' from the CSV file myfile.csv."})
public ModuloCSVLineToUUID(String filename, String fieldname) {
this.filename = filename;
CSVParser csvp = VirtDataResources.readFileCSV(filename);
CSVParser csvp = NBPaths.readFileCSV(filename);
int column = csvp.getHeaderMap().get(fieldname);
for (CSVRecord strings : csvp) {
lines.add(strings.get(column));

4
docsys/src/main/java/io/nosqlbench/docsys/api/Docs.java
View File

@@ -1,6 +1,6 @@
package io.nosqlbench.docsys.api;
import io.nosqlbench.nb.api.pathutil.VirtDataResources;
import io.nosqlbench.nb.api.pathutil.NBPaths;
import java.nio.file.Files;
import java.nio.file.Path;
@@ -25,7 +25,7 @@ public class Docs implements DocsBinder {
}
public Docs addFirstFoundPath(String... potentials) {
Path pathIn = VirtDataResources.findPathIn(potentials);
Path pathIn = NBPaths.findPathIn(potentials);
if (pathIn == null || !Files.exists(pathIn)) {
throw new RuntimeException("Unable to find a path in one of " + Arrays.stream(potentials).collect(Collectors.joining(",")));
}

4
engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLIScenarioParser.java
View File

@@ -5,7 +5,7 @@ import io.nosqlbench.engine.api.activityconfig.StatementsLoader;
import io.nosqlbench.engine.api.activityconfig.yaml.Scenarios;
import io.nosqlbench.engine.api.activityconfig.yaml.StmtsDocList;
import io.nosqlbench.engine.api.exceptions.BasicError;
import io.nosqlbench.nb.api.pathutil.VirtDataResources;
import io.nosqlbench.nb.api.pathutil.NBPaths;
import io.nosqlbench.nb.api.pathutil.NBFiles;
import io.nosqlbench.engine.api.util.StrInterpolator;
import org.slf4j.Logger;
@@ -236,7 +236,7 @@ public class NBCLIScenarioParser {
String dir = "activities/";
Path basePath = VirtDataResources.findPathIn(dir);
Path basePath = NBPaths.findPathIn(dir);
List<Path> yamlPathList = PathWalker.findAll(basePath)
.stream()
.filter(f -> f.toString().endsWith(".yaml"))

4
engine-cli/src/test/java/io/nosqlbench/engine/cli/TestNBCLIOptions.java
View File

@@ -1,7 +1,7 @@
package io.nosqlbench.engine.cli;
import io.nosqlbench.docsys.core.PathWalker;
import io.nosqlbench.nb.api.pathutil.VirtDataResources;
import io.nosqlbench.nb.api.pathutil.NBPaths;
import org.junit.Test;
import java.net.URL;
@@ -205,7 +205,7 @@ public class TestNBCLIOptions {
String dir= "./";
URL resource = getClass().getClassLoader().getResource(dir);
assertThat(resource);
Path basePath = VirtDataResources.findPathIn(dir);
Path basePath = NBPaths.findPathIn(dir);
List<Path> yamlPathList = PathWalker.findAll(basePath).stream().filter(f -> f.toString().endsWith(".yaml")).collect(Collectors.toList());
assertThat(yamlPathList);
}

380
nb-api/src/main/java/io/nosqlbench/nb/api/pathutil/VirtDataResources.java → nb-api/src/main/java/io/nosqlbench/nb/api/pathutil/NBPaths.java
View File

@@ -19,12 +19,11 @@ package io.nosqlbench.nb.api.pathutil;
import org.apache.commons.csv.CSVFormat;
import org.apache.commons.csv.CSVParser;
import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.LogManager;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.*;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
import java.nio.CharBuffer;
import java.nio.file.*;
@@ -33,175 +32,11 @@ import java.nio.file.spi.FileSystemProvider;
import java.util.*;
import java.util.stream.Collectors;
public class VirtDataResources {
public class NBPaths {
private final static Logger logger = LoggerFactory.getLogger(NBPaths.class);
public final static String DATA_DIR = "data";
private final static Logger logger = LogManager.getLogger(VirtDataResources.class);
public static CharBuffer readDataFileToCharBuffer(String basename) {
return loadFileToCharBuffer(basename, DATA_DIR);
}
public static List<String> readDataFileLines(String basename) {
return readFileLines(basename, DATA_DIR);
}
public static String readDataFileString(String basename) {
return readFileString(basename, DATA_DIR);
}
public static InputStream findRequiredStreamOrFile(String basename, String extension, String... searchPaths) {
Optional<InputStream> optionalStreamOrFile = findOptionalStreamOrFile(basename, extension, searchPaths);
return optionalStreamOrFile.orElseThrow(() -> new RuntimeException(
"Unable to find " + basename + " with extension " + extension + " in file system or in classpath, with"
+ " search paths: " + Arrays.stream(searchPaths).collect(Collectors.joining(","))
));
}
public static Reader findRequiredReader(String basename, String extension, String... searchPaths) {
Optional<Reader> optionalReader = findOptionalReader(basename, extension, searchPaths);
return optionalReader.orElseThrow(() -> new RuntimeException(
"Unable to find " + basename + " with extension " + extension + " in file system or in classpath, with"
+ " search paths: " + Arrays.stream(searchPaths).collect(Collectors.joining(","))
));
}
public static Optional<Reader> findOptionalReader(String basename, String extenion, String... searchPaths) {
return findOptionalStreamOrFile(basename, extenion, searchPaths)
.map(InputStreamReader::new)
.map(BufferedReader::new);
}
public static Optional<Path> findOptionalDirPath(String pathName) {
URL systemResource = ClassLoader.getSystemResource(pathName);
if (systemResource != null) {
try {
return Optional.of(Path.of(systemResource.toURI()));
} catch (URISyntaxException e) {
throw new RuntimeException(e);
}
}
return Optional.empty();
}
public static Optional<InputStream> findOptionalStreamOrFile(String basename, String extension, String... searchPaths) {
boolean needsExtension = (extension != null && !extension.isEmpty() && !basename.endsWith("." + extension));
String filename = basename + (needsExtension ? "." + extension : "");
ArrayList<String> paths = new ArrayList<String>() {{
add(filename);
if (!isRemote(basename)) {
addAll(Arrays.stream(searchPaths).map(s -> s + File.separator + filename)
.collect(Collectors.toCollection(ArrayList::new)));
}
}};
for (String path : paths) {
Optional<InputStream> stream = getInputStream(path);
if (stream.isPresent()) {
return stream;
}
}
return Optional.empty();
}
private static boolean isRemote(String path) {
return (path.toLowerCase().startsWith("http:")
|| path.toLowerCase().startsWith("https:"));
}
public static Optional<InputStream> getInputStream(String path) {
// URLs, if http: or https:
if (isRemote(path)) {
Optional<InputStream> inputStream = getInputStreamForUrl(path);
if (inputStream != null) return inputStream;
}
// Files
try {
InputStream stream = new FileInputStream(path);
return Optional.of(stream);
} catch (FileNotFoundException ignored) {
}
// Classpath
ClassLoader classLoader = VirtDataResources.class.getClassLoader();
InputStream stream = classLoader.getResourceAsStream(path);
if (stream != null) {
return Optional.of(stream);
}
return Optional.empty();
}
public static Optional<InputStream> getInputStreamForUrl(String path) {
URL url;
try {
url = new URL(path);
InputStream inputStream = url.openStream();
if (inputStream != null) {
return Optional.of(inputStream);
}
} catch (Exception e) {
throw new RuntimeException(e);
}
return null;
}
public static List<String> readFileLines(String basename, String... searchPaths) {
InputStream requiredStreamOrFile = findRequiredStreamOrFile(basename, "", DATA_DIR);
try (BufferedReader buffer = new BufferedReader((new InputStreamReader(requiredStreamOrFile)))) {
List<String> collected = buffer.lines().collect(Collectors.toList());
return collected;
} catch (IOException ioe) {
throw new RuntimeException("Error while reading required file to string", ioe);
}
}
public static String readFileString(String basename, String... searchPaths) {
InputStream requiredStreamOrFile = findRequiredStreamOrFile(basename, "", searchPaths);
try (BufferedReader buffer = new BufferedReader((new InputStreamReader(requiredStreamOrFile)))) {
String filedata = buffer.lines().collect(Collectors.joining("\n"));
return filedata;
} catch (IOException ioe) {
throw new RuntimeException("Error while reading required file to string", ioe);
}
}
public static CSVParser readFileCSV(String basename, String... searchPaths) {
Reader reader = findRequiredReader(basename, "csv", searchPaths);
CSVFormat format = CSVFormat.newFormat(',').withFirstRecordAsHeader();
try {
CSVParser parser = new CSVParser(reader, format);
return parser;
} catch (IOException e) {
throw new RuntimeException(e);
}
}
public static CharBuffer loadFileToCharBuffer(String filename, String... searchPaths) {
InputStream stream = findRequiredStreamOrFile(filename, "", searchPaths);
CharBuffer linesImage;
try {
InputStreamReader isr = new InputStreamReader(stream);
linesImage = CharBuffer.allocate(1024 * 1024);
while (isr.read(linesImage) > 0) {
}
isr.close();
} catch (IOException e) {
logger.error(e.getMessage());
throw new RuntimeException(e);
}
linesImage.flip();
return linesImage.asReadOnlyBuffer();
}
/**
* <p>Look in all the provided path specifiers for an extant Path, and return
@@ -280,6 +115,54 @@ public class VirtDataResources {
return Optional.ofNullable(foundPath);
}
public static Optional<InputStream> getInputStreamForUrl(String path) {
URL url;
try {
url = new URL(path);
InputStream inputStream = url.openStream();
if (inputStream != null) {
return Optional.of(inputStream);
}
} catch (Exception e) {
throw new RuntimeException(e);
}
return null;
}
public static CSVParser readDelimFile(String basename, char delimiter, String... searchPaths) {
Reader reader = findRequiredReader(basename, "csv", searchPaths);
CSVFormat format = CSVFormat.newFormat(delimiter).withFirstRecordAsHeader();
try {
CSVParser parser = new CSVParser(reader, format);
return parser;
} catch (IOException e) {
throw new RuntimeException(e);
}
}
public static Reader findRequiredReader(String basename, String extension, String... searchPaths) {
Optional<Reader> optionalReader = findOptionalReader(basename, extension, searchPaths);
return optionalReader.orElseThrow(() -> new RuntimeException(
"Unable to find " + basename + " with extension " + extension + " in file system or in classpath, with"
+ " search paths: " + Arrays.stream(searchPaths).collect(Collectors.joining(","))
));
}
public static List<String> readFileLines(String basename, String... searchPaths) {
InputStream requiredStreamOrFile = findRequiredStreamOrFile(basename, "", DATA_DIR);
try (BufferedReader buffer = new BufferedReader((new InputStreamReader(requiredStreamOrFile)))) {
List<String> collected = buffer.lines().collect(Collectors.toList());
return collected;
} catch (IOException ioe) {
throw new RuntimeException("Error while reading required file to string", ioe);
}
}
public static Optional<Reader> findOptionalReader(String basename, String extenion, String... searchPaths) {
return findOptionalStreamOrFile(basename, extenion, searchPaths)
.map(InputStreamReader::new)
.map(BufferedReader::new);
}
private synchronized static Path getPathInFilesystem(URI uri) {
FileSystem fileSystem = null;
@@ -295,9 +178,120 @@ public class VirtDataResources {
return Path.of(uri);
}
public static CSVParser readDelimFile(String basename, char delimiter, String... searchPaths) {
public static InputStream findRequiredStreamOrFile(String basename, String extension, String... searchPaths) {
Optional<InputStream> optionalStreamOrFile = findOptionalStreamOrFile(basename, extension, searchPaths);
return optionalStreamOrFile.orElseThrow(() -> new RuntimeException(
"Unable to find " + basename + " with extension " + extension + " in file system or in classpath, with"
+ " search paths: " + Arrays.stream(searchPaths).collect(Collectors.joining(","))
));
}
public static Optional<InputStream> findOptionalStreamOrFile(String basename, String extension, String... searchPaths) {
boolean needsExtension = (extension != null && !extension.isEmpty() && !basename.endsWith("." + extension));
String filename = basename + (needsExtension ? "." + extension : "");
ArrayList<String> paths = new ArrayList<String>() {{
add(filename);
if (!isRemote(basename)) {
addAll(Arrays.stream(searchPaths).map(s -> s + File.separator + filename)
.collect(Collectors.toCollection(ArrayList::new)));
}
}};
for (String path : paths) {
Optional<InputStream> stream = getInputStream(path);
if (stream.isPresent()) {
return stream;
}
}
return Optional.empty();
}
/**
* Search for the path
*
* @param basename Basename of path, with or without extension
* @param extension The extension of the filename
* @param searchWithin If enabled, all searchPaths are traversed, looking for a matching suffix pattern.
* @param searchPaths Additional places to look for the path suffix
* @return An optional path
*/
public static Optional<Path> findOptionalPath(String basename, String extension, boolean searchWithin, String... searchPaths) {
boolean needsExtension = (extension != null && !extension.isEmpty() && !basename.endsWith("." + extension));
String filename = basename + (needsExtension ? "." + extension : "");
ArrayList<String> paths = new ArrayList<String>() {{
add(filename);
if (!isRemote(basename)) {
addAll(Arrays.stream(searchPaths).map(s -> s + File.separator + filename)
.collect(Collectors.toCollection(ArrayList::new)));
}
}};
for (String path : paths) {
Optional<InputStream> stream = getInputStream(path);
if (stream.isPresent()) {
return Optional.of(Path.of(path));
}
}
if (searchWithin) {
throw new RuntimeException("not implemented");
// for (String searchPath : searchPaths) {
// NBPathWalker.findEndMatching(Path.of(searchPath), Path.of(filename));
// }
}
return Optional.empty();
}
private static boolean isRemote(String path) {
return (path.toLowerCase().startsWith("http:")
|| path.toLowerCase().startsWith("https:"));
}
public static Optional<InputStream> getInputStream(String path) {
// URLs, if http: or https:
if (isRemote(path)) {
URL url;
try {
url = new URL(path);
InputStream inputStream = url.openStream();
if (inputStream != null) {
return Optional.of(inputStream);
}
} catch (Exception e) {
throw new RuntimeException(e);
}
}
// Files
try {
InputStream stream = new FileInputStream(path);
return Optional.of(stream);
} catch (FileNotFoundException ignored) {
}
// Classpath
ClassLoader classLoader = NBPaths.class.getClassLoader();
InputStream stream = classLoader.getResourceAsStream(path);
if (stream != null) {
return Optional.of(stream);
}
return Optional.empty();
}
public static CSVParser readFileCSV(String basename, String... searchPaths) {
Reader reader = findRequiredReader(basename, "csv", searchPaths);
CSVFormat format = CSVFormat.newFormat(delimiter).withFirstRecordAsHeader();
CSVFormat format = CSVFormat.newFormat(',').withFirstRecordAsHeader();
try {
CSVParser parser = new CSVParser(reader, format);
return parser;
@@ -305,4 +299,44 @@ public class VirtDataResources {
throw new RuntimeException(e);
}
}
public static List<String> readDataFileLines(String basename) {
return readFileLines(basename, DATA_DIR);
}
public static String readFile(String basename) {
InputStream requiredStreamOrFile = findRequiredStreamOrFile(basename, "");
try (BufferedReader buffer = new BufferedReader((new InputStreamReader(requiredStreamOrFile)))) {
String filedata = buffer.lines().collect(Collectors.joining("\n"));
return filedata;
} catch (IOException ioe) {
throw new RuntimeException("Error while reading required file to string", ioe);
}
}
public static CharBuffer readDataFileToCharBuffer(String basename) {
return loadFileToCharBuffer(basename, DATA_DIR);
}
public static CharBuffer loadFileToCharBuffer(String filename, String... searchPaths) {
InputStream stream = findRequiredStreamOrFile(filename, "", searchPaths);
CharBuffer linesImage;
try {
InputStreamReader isr = new InputStreamReader(stream);
linesImage = CharBuffer.allocate(1024 * 1024);
while (isr.read(linesImage) > 0) {
}
isr.close();
} catch (IOException e) {
logger.error(e.getMessage());
throw new RuntimeException(e);
}
linesImage.flip();
return linesImage.asReadOnlyBuffer();
}
}

4
virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/core/lfsrs/MetaShift.java
View File

@@ -1,6 +1,6 @@
package io.nosqlbench.virtdata.library.basics.core.lfsrs;
import io.nosqlbench.nb.api.pathutil.VirtDataResources;
import io.nosqlbench.nb.api.pathutil.NBPaths;
import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.LogManager;
@@ -141,7 +141,7 @@ public class MetaShift {
}
int availableSize= Math.max(registerSize,4);
String maskFileName= String.valueOf(availableSize)+"."+"txt";
List<String> lines = VirtDataResources.readDataFileLines("lfsrmasks/" + maskFileName);
List<String> lines = NBPaths.readDataFileLines("lfsrmasks/" + maskFileName);
long[] longs = lines.stream().mapToLong(s -> Long.parseLong(s, 16)).toArray();
return longs;
}

4
virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/distributions/CSVFrequencySampler.java
View File

@@ -14,7 +14,7 @@ import io.nosqlbench.virtdata.api.annotations.ThreadSafeMapper;
import io.nosqlbench.virtdata.library.basics.core.stathelpers.AliasSamplerDoubleInt;
import io.nosqlbench.virtdata.library.basics.core.stathelpers.EvProbD;
import io.nosqlbench.virtdata.library.basics.shared.from_long.to_long.Hash;
import io.nosqlbench.nb.api.pathutil.VirtDataResources;
import io.nosqlbench.nb.api.pathutil.NBPaths;
import org.apache.commons.csv.CSVParser;
import org.apache.commons.csv.CSVRecord;
import org.apache.commons.math3.stat.Frequency;
@@ -72,7 +72,7 @@ public class CSVFrequencySampler implements LongFunction<String> {
Set<String> values = new HashSet<>();
List<EvProbD> frequencies = new ArrayList<>();
CSVParser csvdata = VirtDataResources.readFileCSV(filename);
CSVParser csvdata = NBPaths.readFileCSV(filename);
Frequency freq = new Frequency();
for (CSVRecord csvdatum : csvdata) {
String value = csvdatum.get(columnName);

4
virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/distributions/DelimFrequencySampler.java
View File

@@ -14,7 +14,7 @@ import io.nosqlbench.virtdata.api.annotations.ThreadSafeMapper;
import io.nosqlbench.virtdata.library.basics.core.stathelpers.AliasSamplerDoubleInt;
import io.nosqlbench.virtdata.library.basics.core.stathelpers.EvProbD;
import io.nosqlbench.virtdata.library.basics.shared.from_long.to_long.Hash;
import io.nosqlbench.nb.api.pathutil.VirtDataResources;
import io.nosqlbench.nb.api.pathutil.NBPaths;
import org.apache.commons.csv.CSVParser;
import org.apache.commons.csv.CSVRecord;
import org.apache.commons.math3.stat.Frequency;
@@ -75,7 +75,7 @@ public class DelimFrequencySampler implements LongFunction<String> {
Set<String> values = new HashSet<>();
List<EvProbD> frequencies = new ArrayList<>();
CSVParser csvdata = VirtDataResources.readDelimFile(filename, delimiter);
CSVParser csvdata = NBPaths.readDelimFile(filename, delimiter);
Frequency freq = new Frequency();
for (CSVRecord csvdatum : csvdata) {
String value = csvdatum.get(columnName);

4
virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/distributions/WeightedStringsFromCSV.java
View File

@@ -6,7 +6,7 @@ import io.nosqlbench.virtdata.api.annotations.ThreadSafeMapper;
import io.nosqlbench.virtdata.library.basics.core.stathelpers.AliasSamplerDoubleInt;
import io.nosqlbench.virtdata.library.basics.shared.from_long.to_long.Hash;
import io.nosqlbench.virtdata.library.basics.core.stathelpers.EvProbD;
import io.nosqlbench.nb.api.pathutil.VirtDataResources;
import io.nosqlbench.nb.api.pathutil.NBPaths;
import org.apache.commons.csv.CSVParser;
import org.apache.commons.csv.CSVRecord;
@@ -71,7 +71,7 @@ public class WeightedStringsFromCSV implements LongFunction<String> {
this.hash=new Hash();
}
for (String filename: filenames) {
CSVParser csvdata = VirtDataResources.readFileCSV(filename);
CSVParser csvdata = NBPaths.readFileCSV(filename);
for (CSVRecord csvdatum : csvdata) {
String value = csvdatum.get(valueColumn);
values.add(value);

6
virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_int/HashedLineToInt.java
View File

@@ -1,13 +1,15 @@
package io.nosqlbench.virtdata.library.basics.shared.from_long.to_int;
import io.nosqlbench.virtdata.api.annotations.ThreadSafeMapper;
import io.nosqlbench.nb.api.pathutil.VirtDataResources;
import io.nosqlbench.nb.api.pathutil.NBPaths;
import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.LogManager;
import java.util.List;
import java.util.function.LongToIntFunction;
import static io.nosqlbench.nb.api.pathutil.NBPaths.readDataFileLines;
/**
* Return a pseudo-randomly selected integer value from a file of numeric values.
* Each line in the file must contain one parsable integer value.
@@ -21,7 +23,7 @@ public class HashedLineToInt implements LongToIntFunction {
public HashedLineToInt(String filename) {
this.filename = filename;
List<String> lines = VirtDataResources.readDataFileLines(filename);
List<String> lines = readDataFileLines(filename);
this.values = lines.stream().mapToInt(Integer::parseInt).toArray();
this.intHash = new Hash();
}

4
virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_string/HashedFileExtractToString.java
View File

@@ -21,7 +21,7 @@ package io.nosqlbench.virtdata.library.basics.shared.from_long.to_string;
import io.nosqlbench.virtdata.api.annotations.Example;
import io.nosqlbench.virtdata.api.annotations.ThreadSafeMapper;
import io.nosqlbench.virtdata.library.basics.shared.from_long.to_int.HashRange;
import io.nosqlbench.nb.api.pathutil.VirtDataResources;
import io.nosqlbench.nb.api.pathutil.NBPaths;
import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.LogManager;
@@ -60,7 +60,7 @@ public class HashedFileExtractToString implements LongFunction<String> {
if (fileDataImage == null) {
synchronized (HashedFileExtractToString.class) {
if (fileDataImage == null) {
CharBuffer image= VirtDataResources.readDataFileToCharBuffer(fileName);
CharBuffer image= NBPaths.readDataFileToCharBuffer(fileName);
fileDataImage = image;
}
}

8
virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_string/HashedLineToString.java
View File

@@ -20,7 +20,7 @@ package io.nosqlbench.virtdata.library.basics.shared.from_long.to_string;
import io.nosqlbench.virtdata.api.annotations.ThreadSafeMapper;
import io.nosqlbench.virtdata.library.basics.shared.from_long.to_int.HashRange;
import io.nosqlbench.nb.api.pathutil.VirtDataResources;
import io.nosqlbench.nb.api.pathutil.NBPaths;
import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.LogManager;
@@ -28,6 +28,8 @@ import java.util.ArrayList;
import java.util.List;
import java.util.function.LongFunction;
import static io.nosqlbench.nb.api.pathutil.NBPaths.*;
/**
* Return a pseudo-randomly selected String value from a single line of
* the specified file.
@@ -43,8 +45,8 @@ public class HashedLineToString implements LongFunction<String> {
public HashedLineToString(String filename) {
this.filename = filename;
this.lines = VirtDataResources.readDataFileLines(filename);
this.indexRange = new HashRange(0, lines.size()-2);
this.lines = readDataFileLines(filename);
this.indexRange = new HashRange(0, lines.size() - 2);
}
public String toString() {

4
virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_string/ModuloCSVLineToString.java
View File

@@ -20,7 +20,7 @@ package io.nosqlbench.virtdata.library.basics.shared.from_long.to_string;
import io.nosqlbench.virtdata.api.annotations.Example;
import io.nosqlbench.virtdata.api.annotations.ThreadSafeMapper;
import io.nosqlbench.nb.api.pathutil.VirtDataResources;
import io.nosqlbench.nb.api.pathutil.NBPaths;
import org.apache.commons.csv.CSVParser;
import org.apache.commons.csv.CSVRecord;
import org.apache.logging.log4j.Logger;
@@ -47,7 +47,7 @@ public class ModuloCSVLineToString implements LongFunction<String> {
@Example({"ModuloCSVLineToString('data/myfile.csv','lat')","load values for 'lat' from the CSV file myfile.csv."})
public ModuloCSVLineToString(String filename, String fieldname) {
this.filename = filename;
CSVParser csvp = VirtDataResources.readFileCSV(filename);
CSVParser csvp = NBPaths.readFileCSV(filename);
Map<String, Integer> headerMap = csvp.getHeaderMap();
if (headerMap==null || headerMap.isEmpty()) {

6
virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_string/ModuloLineToString.java
View File

@@ -19,7 +19,7 @@
package io.nosqlbench.virtdata.library.basics.shared.from_long.to_string;
import io.nosqlbench.virtdata.api.annotations.ThreadSafeMapper;
import io.nosqlbench.nb.api.pathutil.VirtDataResources;
import io.nosqlbench.nb.api.pathutil.NBPaths;
import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.LogManager;
@@ -27,6 +27,8 @@ import java.util.ArrayList;
import java.util.List;
import java.util.function.LongFunction;
import static io.nosqlbench.nb.api.pathutil.NBPaths.readDataFileLines;
/**
* Select a value from a text file line by modulo division against the number
* of lines in the file.
@@ -40,7 +42,7 @@ public class ModuloLineToString implements LongFunction<String> {
public ModuloLineToString(String filename) {
this.filename = filename;
this.lines = VirtDataResources.readDataFileLines(filename);
this.lines = readDataFileLines(filename);
}
@Override

4
virtdata-lib-random/src/main/java/io/nosqlbench/virtdata/library/random/RandomFileExtractToString.java
View File

@@ -19,7 +19,7 @@
package io.nosqlbench.virtdata.library.random;
import io.nosqlbench.virtdata.api.annotations.DeprecatedFunction;
import io.nosqlbench.nb.api.pathutil.VirtDataResources;
import io.nosqlbench.nb.api.pathutil.NBPaths;
import org.apache.commons.math3.distribution.IntegerDistribution;
import org.apache.commons.math3.distribution.UniformIntegerDistribution;
import org.apache.commons.math3.random.MersenneTwister;
@@ -60,7 +60,7 @@ public class RandomFileExtractToString implements LongFunction<String> {
if (fileDataImage == null) {
synchronized (RandomFileExtractToString.class) {
if (fileDataImage == null) {
CharBuffer image= VirtDataResources.readDataFileToCharBuffer(fileName);
CharBuffer image= NBPaths.readDataFileToCharBuffer(fileName);
fileDataImage = image;
}
}

4
virtdata-lib-random/src/main/java/io/nosqlbench/virtdata/library/random/RandomLineToInt.java
View File

@@ -1,7 +1,7 @@
package io.nosqlbench.virtdata.library.random;
import io.nosqlbench.virtdata.api.annotations.DeprecatedFunction;
import io.nosqlbench.nb.api.pathutil.VirtDataResources;
import io.nosqlbench.nb.api.pathutil.NBPaths;
import org.apache.commons.math3.distribution.IntegerDistribution;
import org.apache.commons.math3.distribution.UniformIntegerDistribution;
import org.apache.commons.math3.random.MersenneTwister;
@@ -26,7 +26,7 @@ public class RandomLineToInt implements LongToIntFunction {
public RandomLineToInt(String filename, long seed) {
this.filename = filename;
this.lines = VirtDataResources.readDataFileLines(filename);
this.lines = NBPaths.readDataFileLines(filename);
this.rng = new MersenneTwister(seed);
this.itemDistribution= new UniformIntegerDistribution(rng, 0, lines.size()-2);
}

10
virtdata-lib-random/src/main/java/io/nosqlbench/virtdata/library/random/RandomLineToString.java
View File

@@ -19,7 +19,7 @@
package io.nosqlbench.virtdata.library.random;
import io.nosqlbench.virtdata.api.annotations.DeprecatedFunction;
import io.nosqlbench.nb.api.pathutil.VirtDataResources;
import io.nosqlbench.nb.api.pathutil.NBPaths;
import org.apache.commons.math3.distribution.IntegerDistribution;
import org.apache.commons.math3.distribution.UniformIntegerDistribution;
import org.apache.commons.math3.random.MersenneTwister;
@@ -45,22 +45,22 @@ public class RandomLineToString implements LongFunction<String> {
public RandomLineToString(String filename) {
this.rng = new MersenneTwister(System.nanoTime());
this.filename = filename;
this.lines = VirtDataResources.readDataFileLines(filename);
this.lines = NBPaths.readDataFileLines(filename);
itemDistribution= new UniformIntegerDistribution(rng, 0, lines.size()-2);
}
public RandomLineToString(String filename, MersenneTwister rng) {
this.rng = rng;
this.filename = filename;
this.lines = VirtDataResources.readDataFileLines(filename);
this.lines = VirtDataResources.readDataFileLines(filename);
this.lines = NBPaths.readDataFileLines(filename);
this.lines = NBPaths.readDataFileLines(filename);
itemDistribution= new UniformIntegerDistribution(rng, 0, lines.size()-2);
}
public RandomLineToString(String filename, long seed) {
this.rng = new MersenneTwister(seed);
this.filename = filename;
this.lines = VirtDataResources.readDataFileLines(filename);
this.lines = NBPaths.readDataFileLines(filename);
itemDistribution= new UniformIntegerDistribution(rng, 0, lines.size()-2);
}

4
virtdata-userlibs/src/main/java/io/nosqlbench/virtdata/userlibs/apps/docsapp/AutoDocsApp.java
View File

@@ -4,7 +4,7 @@ import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import io.nosqlbench.virtdata.api.annotations.Category;
import io.nosqlbench.virtdata.core.bindings.VirtDataDocs;
import io.nosqlbench.nb.api.pathutil.VirtDataResources;
import io.nosqlbench.nb.api.pathutil.NBPaths;
import io.nosqlbench.virtdata.api.processors.DocFuncData;
import io.nosqlbench.virtdata.userlibs.apps.docsapp.fdocs.FDoc;
import io.nosqlbench.virtdata.userlibs.apps.docsapp.fdocs.FDocCat;
@@ -143,7 +143,7 @@ public class AutoDocsApp implements Runnable {
String[] blurbsdirs = blurbsDirs.split(":");
for (String blurbsdir : blurbsdirs) {
Optional<Path> bdir = VirtDataResources.FindOptionalPathIn(blurbsdir+"/");
Optional<Path> bdir = NBPaths.FindOptionalPathIn(blurbsdir+"/");
if (bdir.isPresent()) {
Path blurbsFile = bdir.get().resolve(Path.of(outputname).getFileName().toString());
if (Files.exists(blurbsFile)) {