IntenseFinance/nosqlbench
3
0
Fork 0
mirror of https://github.com/nosqlbench/nosqlbench.git synced 2025-02-25 18:55:28 -06:00
Code Issues Packages Projects Releases Wiki Activity

partial rework of the NBIO internals for multiple results

Browse Source
This commit is contained in:
Jonathan Shook 2020-04-08 12:38:16 -05:00
parent d8b3e6214c
commit 1e05c31075
38 changed files with 240 additions and 673 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
activitytype-cql/src/main/java/io/nosqlbench/activitytype/cql/datamappers/functions/long_string/ModuloCSVLineToUUID.java
View File

@ -19,9 +19,9 @@
package io.nosqlbench.activitytype.cql.datamappers.functions.long_string;
import io.nosqlbench.nb.api.content.NBIO;
import io.nosqlbench.virtdata.api.annotations.Example;
import io.nosqlbench.virtdata.api.annotations.ThreadSafeMapper;
import io.nosqlbench.nb.api.pathutil.NBPaths;
import io.nosqlbench.virtdata.library.basics.shared.from_long.to_string.ModuloLineToString;
import org.apache.commons.csv.CSVParser;
import org.apache.commons.csv.CSVRecord;
@ -49,7 +49,7 @@ public class ModuloCSVLineToUUID implements LongFunction<UUID> {
@Example({"ModuloCSVLineToUUID('data/myfile.csv','lat')","load values for 'lat' from the CSV file myfile.csv."})
public ModuloCSVLineToUUID(String filename, String fieldname) {
this.filename = filename;
CSVParser csvp = NBPaths.readFileCSV(filename);
CSVParser csvp = NBIO.readFileCSV(filename);
int column = csvp.getHeaderMap().get(fieldname);
for (CSVRecord strings : csvp) {
lines.add(strings.get(column));

13
activitytype-cql/src/main/java/io/nosqlbench/activitytype/cql/statements/core/YamlCQLStatementLoader.java
View File

@ -1,7 +1,8 @@
package io.nosqlbench.activitytype.cql.statements.core;
import io.nosqlbench.engine.api.activityimpl.ActivityInitializationError;
import io.nosqlbench.nb.api.pathutil.NBPaths;
import io.nosqlbench.nb.api.content.Content;
import io.nosqlbench.nb.api.content.NBIO;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.yaml.snakeyaml.TypeDescription;
@ -32,14 +33,8 @@ public class YamlCQLStatementLoader {
public AvailableCQLStatements load(String fromPath, String... searchPaths) {
InputStream stream = NBPaths.findRequiredStreamOrFile(fromPath,
"yaml", searchPaths);
String data = "";
try (BufferedReader buffer = new BufferedReader(new InputStreamReader(stream))) {
data = buffer.lines().collect(Collectors.joining("\n"));
} catch (Exception e) {
throw new RuntimeException("Error while reading yaml stream data:" + e);
}
Content<?> yamlContent = NBIO.all().prefix(searchPaths).name(fromPath).extension("yaml").one();
String data = yamlContent.asString();
for (Function<String, String> xform : transformers) {
try {

5
docsys/src/main/java/io/nosqlbench/docsys/api/Docs.java
View File

@ -1,6 +1,6 @@
package io.nosqlbench.docsys.api;
import io.nosqlbench.nb.api.pathutil.NBPaths;
import io.nosqlbench.nb.api.content.NBIO;
import java.nio.file.Files;
import java.nio.file.Path;
@ -25,7 +25,8 @@ public class Docs implements DocsBinder {
}
public Docs addFirstFoundPath(String... potentials) {
Path pathIn = NBPaths.findPathIn(potentials);
Path pathIn = NBIO.getFirstLocalPath(potentials);
if (pathIn == null || !Files.exists(pathIn)) {
throw new RuntimeException("Unable to find a path in one of " + Arrays.stream(potentials).collect(Collectors.joining(",")));
}

6
engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/StatementsLoader.java
View File

@ -46,15 +46,11 @@ public class StatementsLoader {
// }
public static StmtsDocList load(Logger logger, String path, String... searchPaths) {
Optional<Content<?>> foundStmtsDoc = NBIO.all()
Content<?> content = NBIO.all()
.prefix(searchPaths)
.name(path)
.one();
Content<?> content = foundStmtsDoc.orElseThrow(() ->
new BasicError("Unable to find content for '" + path + "' in search paths: " + String.join(",",
searchPaths)));
RawYamlStatementLoader loader = new RawYamlStatementLoader();
RawStmtsDocList rawDocList = loader.loadString(logger, content.get());
StmtsDocList layered = new StmtsDocList(rawDocList);

18
engine-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawYamlStatementLoader.java
View File

@ -20,7 +20,8 @@ package io.nosqlbench.engine.api.activityconfig.rawyaml;
import io.nosqlbench.engine.api.activityconfig.snakecharmer.SnakeYamlCharmer;
import io.nosqlbench.engine.api.activityimpl.ActivityInitializationError;
import io.nosqlbench.nb.api.content.Content;
import io.nosqlbench.nb.api.pathutil.NBPaths;
import io.nosqlbench.nb.api.content.NBIO;
import io.nosqlbench.nb.api.errors.BasicError;
import org.slf4j.Logger;
import org.yaml.snakeyaml.TypeDescription;
import org.yaml.snakeyaml.Yaml;
@ -34,6 +35,7 @@ import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Optional;
import java.util.function.Function;
import java.util.stream.Collectors;
@ -49,7 +51,8 @@ public class RawYamlStatementLoader {
}
public RawStmtsDocList load(Logger logger, String fromPath, String... searchPaths) {
String data = loadRawFile(logger, fromPath, searchPaths);
Optional<Content<?>> oyaml = NBIO.all().prefix(searchPaths).name(fromPath).extension("yaml").first();
String data = oyaml.map(Content::asString).orElseThrow(() -> new BasicError("Unable to load " + fromPath));
data = applyTransforms(logger, data);
return parseYaml(logger, data);
}
@ -67,17 +70,6 @@ public class RawYamlStatementLoader {
}
}
protected String loadRawFile(Logger logger, String fromPath, String... searchPaths) {
InputStream stream = NBPaths.findRequiredStreamOrFile(fromPath, "yaml", searchPaths);
try (BufferedReader buffer = new BufferedReader(new InputStreamReader(stream))) {
return buffer.lines().collect(Collectors.joining("\n"));
} catch (Exception e) {
throw new RuntimeException(
"Error while reading YAML from search paths:" + Arrays.toString(searchPaths) + ":" + e.getMessage(), e
);
}
}
protected String applyTransforms(Logger logger, String data) {
for (Function<String, String> xform : stringTransformers) {
try {

7
engine-api/src/main/java/io/nosqlbench/engine/api/scenarios/NBCLIScenarioParser.java
View File

@ -84,7 +84,7 @@ public class NBCLIScenarioParser {
for (String scenarioName : scenarioNames) {
// Load in named scenario
Optional<Content<?>> yamlWithNamedScenarios = NBIO.all().prefix(SEARCH_IN)
Content<?> yamlWithNamedScenarios = NBIO.all().prefix(SEARCH_IN)
.name(workloadName)
.extension("yaml")
.one();
@ -94,7 +94,7 @@ public class NBCLIScenarioParser {
// StmtsDocList stmts = StatementsLoader.load(logger, workloadName, SEARCH_IN);
StmtsDocList stmts = StatementsLoader.load(logger,yamlWithNamedScenarios.get());
StmtsDocList stmts = StatementsLoader.load(logger,yamlWithNamedScenarios);
Scenarios scenarios = stmts.getDocScenarios();
@ -282,11 +282,10 @@ public class NBCLIScenarioParser {
for (Path yamlPath : yamlPathList) {
String referencedWorkloadName = yamlPath.toString().substring(1);
Optional<Content<?>> referencedWorkload = NBIO.all().prefix(SEARCH_IN)
Content<?> content = NBIO.all().prefix(SEARCH_IN)
.name(referencedWorkloadName).extension("yaml")
.one();
Content<?> content = referencedWorkload.orElseThrow();
StmtsDocList stmts = StatementsLoader.load(logger,content);
Set<String> templates = new HashSet<>();

49
engine-api/src/test/java/io/nosqlbench/engine/api/util/NBPathsTest.java
View File

@ -1,49 +0,0 @@
/*
*
* Copyright 2016 jshook
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* /
*/
package io.nosqlbench.engine.api.util;
import io.nosqlbench.nb.api.pathutil.NBPaths;
import org.junit.Test;
import java.io.InputStream;
import java.util.Optional;
import static org.assertj.core.api.Assertions.assertThat;
public class NBPathsTest {
@Test
public void testNestedClasspathLoading() {
Optional<InputStream> optionalStreamOrFile = NBPaths.findOptionalStreamOrFile("nested/testfile", "txt", "activities");
assertThat(optionalStreamOrFile).isPresent();
}
// @Test
// public void testUrlResourceSearchSanity() {
// String url="https://google.com/robots";
// Optional<InputStream> inputStream = NosqlBenchFiles.findOptionalStreamOrFile(url,"txt","activity");
// assertThat(inputStream).isPresent();
// }
//
// @Test
// public void testUrlResourceLoading() {
// String url="https://google.com/";
// Optional<InputStream> inputStream = NosqlBenchFiles.getInputStream(url);
// assertThat(inputStream).isPresent();
// }
}

13
engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLIOptions.java
View File

@ -3,8 +3,9 @@ package io.nosqlbench.engine.cli;
import ch.qos.logback.classic.Level;
import io.nosqlbench.engine.api.metrics.IndicatorMode;
import io.nosqlbench.engine.api.scenarios.NBCLIScenarioParser;
import io.nosqlbench.nb.api.pathutil.NBPaths;
import io.nosqlbench.engine.api.util.Unit;
import io.nosqlbench.nb.api.content.Content;
import io.nosqlbench.nb.api.content.NBIO;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -322,10 +323,14 @@ public class NBCLIOptions {
wantsToCopyWorkload = readWordOrThrow(arglist, "workload to copy");
break;
default:
Optional<InputStream> optionalScript =
NBPaths.findOptionalStreamOrFile(word, "js", "scripts/auto");
Optional<Content<?>> scriptfile = NBIO.local()
.prefix("scripts/auto")
.name(word)
.extension("js")
.first();
//Script
if (optionalScript.isPresent()) {
if (scriptfile.isPresent()) {
arglist.removeFirst();
arglist.addFirst("scripts/auto/" + word);
arglist.addFirst("script");

11
engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLIScriptAssembly.java
View File

@ -1,8 +1,9 @@
package io.nosqlbench.engine.cli;
import io.nosqlbench.engine.api.activityimpl.ActivityDef;
import io.nosqlbench.nb.api.pathutil.NBPaths;
import io.nosqlbench.engine.api.util.StrInterpolator;
import io.nosqlbench.nb.api.content.Content;
import io.nosqlbench.nb.api.content.NBIO;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -81,13 +82,9 @@ public class NBCLIScriptAssembly {
} catch (IOException ignored) {
}
InputStream resourceAsStream = NBPaths.findRequiredStreamOrFile(cmd.getCmdSpec(), "js", "scripts");
Content<?> one = NBIO.all().prefix("scripts").name(cmd.getCmdSpec()).extension("js").one();
scriptData = one.asString();
try (BufferedReader buffer = new BufferedReader(new InputStreamReader(resourceAsStream))) {
scriptData = buffer.lines().collect(Collectors.joining("\n"));
} catch (Throwable t) {
throw new RuntimeException("Unable to buffer " + cmd.getCmdSpec() + ": " + t);
}
StrInterpolator interpolator = new StrInterpolator(cmd.getCmdArgs());
scriptData = interpolator.apply(scriptData);
return new ScriptData(scriptData,cmd.getCmdArgs());

4
engine-cli/src/test/java/io/nosqlbench/engine/cli/TestNBCLIOptions.java
View File

@ -1,7 +1,7 @@
package io.nosqlbench.engine.cli;
import io.nosqlbench.docsys.core.PathWalker;
import io.nosqlbench.nb.api.pathutil.NBPaths;
import io.nosqlbench.nb.api.content.NBIO;
import org.junit.Test;
import java.net.URL;
@ -205,7 +205,7 @@ public class TestNBCLIOptions {
String dir= "./";
URL resource = getClass().getClassLoader().getResource(dir);
assertThat(resource);
Path basePath = NBPaths.findPathIn(dir);
Path basePath = NBIO.getFirstLocalPath(dir);
List<Path> yamlPathList = PathWalker.findAll(basePath).stream().filter(f -> f.toString().endsWith(".yaml")).collect(Collectors.toList());
assertThat(yamlPathList);
}

4
engine-docker/src/main/java/io/nosqlbench/engine/docker/DockerMetricsManager.java
View File

@ -12,7 +12,7 @@ import com.github.dockerjava.api.model.ContainerNetworkSettings;
import com.github.dockerjava.api.model.Frame;
import com.github.dockerjava.core.async.ResultCallbackTemplate;
import com.github.dockerjava.core.command.LogContainerResultCallback;
import io.nosqlbench.nb.api.pathutil.NBPaths;
import io.nosqlbench.nb.api.content.NBIO;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -147,7 +147,7 @@ public class DockerMetricsManager {
}
private void setupPromFiles(String ip) {
String datasource = NBPaths.readFile("docker/prometheus/prometheus.yml");
String datasource = NBIO.readCharBuffer("docker/prometheus/prometheus.yml").toString();
if (ip == null) {
logger.error("IP for graphite container not found");

4
engine-docker/src/main/java/io/nosqlbench/engine/docker/RestHelper.java
View File

@ -1,7 +1,7 @@
package io.nosqlbench.engine.docker;
import io.nosqlbench.nb.api.content.NBIO;
import io.nosqlbench.nb.api.errors.BasicError;
import io.nosqlbench.nb.api.pathutil.NBPaths;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -36,7 +36,7 @@ public class RestHelper {
if (path !=null) {
logger.debug("POSTing " + path + " to " + url);
String dashboard = NBPaths.readFile(path);
String dashboard = NBIO.readCharBuffer(path).toString();
logger.debug("length of content for " + path + " is " + dashboard.length());
builder = builder.POST(HttpRequest.BodyPublishers.ofString(dashboard));
builder.setHeader("Content-Type", "application/json");

6
engine-extensions/src/main/java/io/nosqlbench/engine/extensions/files/FileAccess.java
View File

@ -17,13 +17,11 @@
package io.nosqlbench.engine.extensions.files;
import io.nosqlbench.nb.api.pathutil.NBPaths;
import io.nosqlbench.nb.api.content.NBIO;
public class FileAccess extends FileAccessPluginData {
public String read(String filepath) {
String filedata = NBPaths.readFile(filepath);
return filedata;
return NBIO.readCharBuffer(filepath).toString();
}
}

27
nb-api/src/main/java/io/nosqlbench/nb/api/content/Content.java
View File

@ -1,19 +1,27 @@
package io.nosqlbench.nb.api.content;
import java.io.*;
import java.net.URI;
import java.nio.CharBuffer;
import java.nio.file.FileSystem;
import java.nio.file.OpenOption;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.nio.file.spi.FileSystemProvider;
import java.util.function.Supplier;
/**
* A generic content wrapper for anything that can be given to a NoSQLBench runtime
* using a specific type of locator.
*
* @param <T>
*/
public interface Content<T> extends Supplier<CharSequence>, Comparable<Content<?>> {
T getLocation();
URI getURI();
Path asPath();
public default String asString() {
@ -21,6 +29,7 @@ public interface Content<T> extends Supplier<CharSequence>, Comparable<Content<?
}
CharBuffer getCharBuffer();
@Override
default CharSequence get() {
return getCharBuffer();
@ -30,4 +39,22 @@ public interface Content<T> extends Supplier<CharSequence>, Comparable<Content<?
return getURI().compareTo(other.getURI());
}
default Reader getReader() {
InputStream inputStream = getInputStream();
return new InputStreamReader(inputStream);
}
default InputStream getInputStream() {
try {
Path path = asPath();
FileSystem fileSystem = path.getFileSystem();
FileSystemProvider provider = fileSystem.provider();
InputStream stream = provider.newInputStream(path, StandardOpenOption.READ);
return stream;
} catch (IOException ignored) {
}
String stringdata = getCharBuffer().toString();
return new ByteArrayInputStream(stringdata.getBytes());
}
}

11
nb-api/src/main/java/io/nosqlbench/nb/api/content/ContentResolver.java
View File

@ -2,6 +2,7 @@ package io.nosqlbench.nb.api.content;
import java.net.URI;
import java.nio.file.Path;
import java.util.List;
import java.util.Optional;
public interface ContentResolver {
@ -25,15 +26,13 @@ public interface ContentResolver {
* @param uri The URI of a content location, like a file name or URL.
* @return A content element which may then be used to access the content
*/
Content<?> resolve(URI uri);
default Content<?> resolve(String uri) {
List<Content<?>> resolve(URI uri);
default List<Content<?>> resolve(String uri) {
return resolve(URI.create(uri));
}
Optional<Path> resolveDirectory(URI uri);
default Optional<Path> resolveDirectory(String uri) {
List<Path> resolveDirectory(URI uri);
default List<Path> resolveDirectory(String uri) {
return resolveDirectory(URI.create(uri));
}

66
nb-api/src/main/java/io/nosqlbench/nb/api/content/NBIO.java
View File

@ -2,8 +2,13 @@ package io.nosqlbench.nb.api.content;
import io.nosqlbench.nb.api.content.fluent.NBPathsAPI;
import io.nosqlbench.nb.api.errors.BasicError;
import org.apache.commons.csv.CSVFormat;
import org.apache.commons.csv.CSVParser;
import java.io.IOException;
import java.io.InputStream;
import java.io.Reader;
import java.nio.CharBuffer;
import java.nio.file.DirectoryStream;
import java.nio.file.FileSystems;
import java.nio.file.Path;
@ -44,6 +49,56 @@ public class NBIO implements NBPathsAPI.Facets {
this.extensions = extensions;
}
public static List<String> readLines(String filename) {
Content<?> data = NBIO.all().prefix("data").name(filename).one();
String[] split = data.getCharBuffer().toString().split("\n");
return Arrays.asList(split);
}
public static CSVParser readFileCSV(String filename, String... searchPaths) {
return NBIO.readFileDelimCSV(filename, ',', searchPaths);
}
public static CSVParser readFileDelimCSV(String filename,char delim, String... searchPaths) {
Reader reader = NBIO.readReader(filename, searchPaths);
CSVFormat format = CSVFormat.newFormat(delim).withFirstRecordAsHeader();
try {
CSVParser parser = new CSVParser(reader, format);
return parser;
} catch (IOException e) {
throw new RuntimeException(e);
}
}
private static InputStream readInputStream(String filename, String... searchPaths) {
return NBIO.all().prefix(searchPaths).name(filename).one().getInputStream();
}
private static Reader readReader(String filename, String... searchPaths) {
return NBIO.all().prefix(searchPaths).name(filename).one().getReader();
}
public static CharBuffer readCharBuffer(String fileName, String... searchPaths) {
return NBIO.all().prefix(searchPaths).name(fileName).one().getCharBuffer();
}
public static Path getFirstLocalPath(String... potentials) {
Optional<Content<?>> first = NBIO.local().name(potentials).first();
return first.orElseThrow().asPath();
}
public static Optional<Path> findFirstLocalPath(String... potentials) {
Optional<Content<?>> first = NBIO.local().name(potentials).first();
Optional<Path> path = first.map(Content::asPath);
return path;
}
public static InputStream readInputStream(String fromPath, String yaml, String[] searchPaths) {
return null;
}
@Override
public NBPathsAPI.GetPrefix localContent() {
this.resolver = URIResolvers.inFS().inCP();
@ -147,7 +202,7 @@ public class NBIO implements NBPathsAPI.Facets {
public Optional<Content<?>> first() {
List<Content<?>> list = list();
if (list.size()>0) {
if (list.size() > 0) {
return Optional.of(list.get(0));
} else {
return Optional.empty();
@ -166,7 +221,7 @@ public class NBIO implements NBPathsAPI.Facets {
}
@Override
public Optional<Content<?>> one() {
public Content<?> one() {
List<Content<?>> list = list();
@ -178,7 +233,7 @@ public class NBIO implements NBPathsAPI.Facets {
String found = list.stream().map(c -> c.getURI().toString()).collect(Collectors.joining(","));
throw new BasicError(("Found too many sources for '" + this.toString() + "', ambiguous name. Pick from " + found));
}
return Optional.of(list.get(0));
return list.get(0);
}
@ -372,6 +427,11 @@ public class NBIO implements NBPathsAPI.Facets {
public Iterator<Path> iterator() {
return found.iterator();
}
public String toString() {
return "FileCapture{n=" + found.size() + (found.size()>0?"," +found.get(0).toString():"") +"}";
}
}
@Override

5
nb-api/src/main/java/io/nosqlbench/nb/api/content/PathContent.java
View File

@ -59,4 +59,9 @@ public class PathContent implements Content<Path> {
public int hashCode() {
return Objects.hash(path);
}
public String toString() {
return "PathContent{" + getURI().toString() + "}";
}
}

58
nb-api/src/main/java/io/nosqlbench/nb/api/content/ResolverForClasspath.java
View File

@ -4,9 +4,8 @@ import java.io.IOException;
import java.net.URI;
import java.net.URL;
import java.nio.file.*;
import java.nio.file.spi.FileSystemProvider;
import java.util.Collections;
import java.util.Optional;
import java.util.*;
import java.util.stream.Collectors;
/**
* Resolves resources which can be found via the class loader.
@ -23,15 +22,32 @@ public class ResolverForClasspath implements ContentResolver {
public static final ContentResolver INSTANCE = new ResolverForClasspath();
private Path resolvePath(URI uri) {
private List<Path> resolvePaths(URI uri) {
List<Path> paths = new ArrayList<>();
if (uri.getScheme() != null && !uri.getScheme().isEmpty()) {
return null;
}
URL systemResource = ClassLoader.getSystemResource(uri.getPath());
if (systemResource == null) {
// URL systemResource = ClassLoader.getSystemResource(uri.getPath());
try {
Enumeration<URL> systemResources = ClassLoader.getSystemResources(uri.getPath());
while (systemResources.hasMoreElements()) {
URL url = systemResources.nextElement();
Path p = normalize(url);
paths.add(p);
}
} catch (IOException e) {
e.printStackTrace();
}
return paths;
}
private Path normalize(URL url) {
if (url == null) {
return null;
}
URI resolved = URI.create(systemResource.toExternalForm());
URI resolved = URI.create(url.toExternalForm());
if (resolved.getScheme().equals("file")) {
Path current = Paths.get("").toAbsolutePath();
Path logical = Path.of(resolved.getPath());
@ -55,25 +71,23 @@ public class ResolverForClasspath implements ContentResolver {
}
@Override
public Content<?> resolve(URI uri) {
Path path = resolvePath(uri);
if (path == null) {
return null;
}
return new PathContent(path);
public List<Content<?>> resolve(URI uri) {
List<Path> paths = resolvePaths(uri);
List<Content<?>> contents = paths.stream().map(PathContent::new).collect(Collectors.toList());
return contents;
}
@Override
public Optional<Path> resolveDirectory(URI uri) {
Path path = resolvePath(uri);
if (path == null) {
return Optional.empty();
}
if (Files.isDirectory(path)) {
return Optional.of(path);
} else {
return Optional.empty();
public List<Path> resolveDirectory(URI uri) {
List<Path> path = resolvePaths(uri);
List<Path> dirs = new ArrayList<>();
for (Path dirpath : path) {
if (Files.isDirectory(dirpath)) {
dirs.add(dirpath);
}
}
return dirs;
}
public String toString() {

28
nb-api/src/main/java/io/nosqlbench/nb/api/content/ResolverForFilesystem.java
View File

@ -3,6 +3,8 @@ package io.nosqlbench.nb.api.content;
import java.net.URI;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
public class ResolverForFilesystem implements ContentResolver {
@ -10,7 +12,7 @@ public class ResolverForFilesystem implements ContentResolver {
public static ResolverForFilesystem INSTANCE = new ResolverForFilesystem();
private Path resolvePath(URI uri) {
if (uri.getScheme()!=null&&!uri.getScheme().isEmpty()&&!uri.getScheme().equals("file")) {
if (uri.getScheme() != null && !uri.getScheme().isEmpty() && !uri.getScheme().equals("file")) {
return null;
}
Path pathFromUri = Path.of(uri.getPath());
@ -22,25 +24,25 @@ public class ResolverForFilesystem implements ContentResolver {
}
@Override
public Content<?> resolve(URI uri) {
public List<Content<?>> resolve(URI uri) {
List<Content<?>> contents = new ArrayList<>();
Path path = resolvePath(uri);
if (path==null) {
return null;
if (path != null) {
contents.add(new PathContent(path));
}
return new PathContent(path);
return contents;
}
@Override
public Optional<Path> resolveDirectory(URI uri) {
public List<Path> resolveDirectory(URI uri) {
List<Path> dirs = new ArrayList<>();
Path path = resolvePath(uri);
if (path == null) {
return Optional.empty();
}
if (Files.isDirectory(path)) {
return Optional.of(path);
} else {
return Optional.empty();
if (path!=null && Files.isDirectory(path)) {
dirs.add(path);
}
return dirs;
}
public String toString() {

11
nb-api/src/main/java/io/nosqlbench/nb/api/content/ResolverForURL.java
View File

@ -8,6 +8,9 @@ import java.io.InputStream;
import java.net.URI;
import java.net.URL;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Optional;
public class ResolverForURL implements ContentResolver {
@ -16,7 +19,7 @@ public class ResolverForURL implements ContentResolver {
private final static Logger logger = LoggerFactory.getLogger(ResolverForURL.class);
@Override
public Content<?> resolve(URI uri) {
public List<Content<?>> resolve(URI uri) {
if (uri.getScheme()==null) {
return null;
}
@ -26,7 +29,7 @@ public class ResolverForURL implements ContentResolver {
URL url = uri.toURL();
InputStream inputStream = url.openStream();
logger.debug("Found accessible remote file at " + url.toString());
return new URLContent(url, inputStream);
return List.of(new URLContent(url, inputStream));
} catch (IOException e) {
throw new RuntimeException(e);
}
@ -35,8 +38,8 @@ public class ResolverForURL implements ContentResolver {
}
@Override
public Optional<Path> resolveDirectory(URI uri) {
return Optional.empty();
public List<Path> resolveDirectory(URI uri) {
return Collections.emptyList();
}
public String toString() {

30
nb-api/src/main/java/io/nosqlbench/nb/api/content/URIResolver.java
View File

@ -68,32 +68,23 @@ public class URIResolver implements ContentResolver {
return this;
}
public Optional<Content<?>> resolveOptional(String uri) {
return Optional.ofNullable(resolve(uri));
}
public Content<?> resolve(String uri) {
public List<Content<?>> resolve(String uri) {
return resolve(URI.create(uri));
}
@Override
public Optional<Path> resolveDirectory(URI uri) {
public List<Path> resolveDirectory(URI uri) {
List<Path> dirs = new ArrayList<>();
for (ContentResolver loader : loaders) {
Optional<Path> path = loader.resolveDirectory(uri);
if (path.isPresent()) {
return path;
}
dirs.addAll(loader.resolveDirectory(uri));
}
return Optional.empty();
return dirs;
}
public Content<?> resolve(URI uri) {
Content<?> resolved = null;
public List<Content<?>> resolve(URI uri) {
List<Content<?>> resolved = new ArrayList<>();
for (ContentResolver loader : loaders) {
resolved = loader.resolve(uri);
if (resolved!=null) {
break;
}
resolved.addAll(loader.resolve(uri));
}
return resolved;
}
@ -105,10 +96,7 @@ public class URIResolver implements ContentResolver {
public List<Content<?>> resolveAll(URI uri) {
List<Content<?>> allFound = new ArrayList<>();
for (ContentResolver loader : loaders) {
Content<?> found = loader.resolve(uri);
if (found!=null) {
allFound.add(found);
}
allFound.addAll(loader.resolve(uri));
}
return allFound;
}

4
nb-api/src/main/java/io/nosqlbench/nb/api/content/URLContent.java
View File

@ -75,4 +75,8 @@ public class URLContent implements Content<URL> {
public Path asPath() {
return null;
}
public String toString() {
return "URLContent{" + getURI().toString() + "}";
}
}

2
nb-api/src/main/java/io/nosqlbench/nb/api/content/fluent/NBPathsAPI.java
View File

@ -96,7 +96,7 @@ public interface NBPathsAPI {
* It is an error if you find none, or more than one.
* @return An optional content element.
*/
Optional<Content<?>> one();
Content<?> one();
}

125
nb-api/src/main/java/io/nosqlbench/nb/api/pathutil/NBPathWalker.java
View File

@ -1,125 +0,0 @@
package io.nosqlbench.nb.api.pathutil;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.io.IOException;
import java.nio.file.DirectoryStream;
import java.nio.file.Path;
import java.nio.file.attribute.BasicFileAttributes;
import java.nio.file.spi.FileSystemProvider;
import java.util.ArrayList;
import java.util.List;
public class NBPathWalker {
private final static Logger logger = LogManager.getLogger(NBPathWalker.class);
public static void walk(Path p, PathVisitor v) {
walk(p, v, NBPathWalker.WALK_ALL);
}
public static List<Path> findAll(Path p) {
Collect fileCollector = new Collect(true, false);
walk(p, fileCollector);
return fileCollector.get();
}
public static List<Path> findEndMatching(Path p, Path endingName) {
Collect fileCollector = new Collect(true, false);
MatchEnding ending = new MatchEnding(endingName.toString());
walk(p, fileCollector, ending);
return fileCollector.get();
}
public static void walk(Path p, PathVisitor v, DirectoryStream.Filter<Path> filter) {
try {
FileSystemProvider provider = p.getFileSystem().provider();
DirectoryStream<Path> paths = provider.newDirectoryStream(p, (Path r) -> true);
List<Path> pathlist = new ArrayList<>();
for (Path path : paths) {
pathlist.add(path);
}
for (Path path : pathlist) {
if (path.getFileSystem().provider().readAttributes(path, BasicFileAttributes.class).isDirectory()) {
v.preVisitDir(path);
walk(path, v, filter);
v.postVisitDir(path);
} else if (filter.accept(path)) {
v.preVisitFile(path);
v.visit(path);
v.postVisitFile(path);
} else {
logger.error("error");
}
}
} catch (IOException e) {
throw new RuntimeException(e);
}
}
public interface PathVisitor {
void visit(Path p);
default void preVisitFile(Path path) {
}
default void postVisitFile(Path path) {
}
default void preVisitDir(Path path) {
}
default void postVisitDir(Path path) {
}
}
public static DirectoryStream.Filter<Path> WALK_ALL = entry -> true;
public static class MatchEnding implements DirectoryStream.Filter<Path> {
private final String regex;
public MatchEnding(String pathEnd) {
this.regex = pathEnd;
}
@Override
public boolean accept(Path entry) throws IOException {
return entry.toString().endsWith(regex);
}
}
public static class Collect implements PathVisitor {
private final List<Path> listing = new ArrayList<>();
private final boolean collectFiles;
private final boolean collectDirectories;
public Collect(boolean collectFiles, boolean collectDirectories) {
this.collectFiles = collectFiles;
this.collectDirectories = collectDirectories;
}
public List<Path> get() {
return listing;
}
@Override
public void visit(Path p) {
}
@Override
public void preVisitFile(Path path) {
if (this.collectFiles) {
listing.add(path);
}
}
@Override
public void preVisitDir(Path path) {
if (this.collectDirectories) {
listing.add(path);
}
}
}
}

341
nb-api/src/main/java/io/nosqlbench/nb/api/pathutil/NBPaths.java
View File

@ -1,341 +0,0 @@
/*
*
* Copyright 2016 jshook
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* /
*/
package io.nosqlbench.nb.api.pathutil;
import org.apache.commons.csv.CSVFormat;
import org.apache.commons.csv.CSVParser;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.*;
import java.net.URI;
import java.net.URL;
import java.nio.CharBuffer;
import java.nio.file.*;
import java.nio.file.attribute.BasicFileAttributes;
import java.nio.file.spi.FileSystemProvider;
import java.util.*;
import java.util.stream.Collectors;
public class NBPaths {
private final static Logger logger = LoggerFactory.getLogger(NBPaths.class);
public final static String DATA_DIR = "data";
/**
* <p>Look in all the provided path specifiers for an extant Path, and return
* the first one found.</p>
*
* <p>If the final character of any path specifier is the default file
* separator, then the request is for a directory. During searching,
* if a directory is found when a file is requested, or vice-versa, then
* an error is thrown withouth looking further.</p>
*
* <p>The locations that are searched include:</p>
* <OL>
* <LI>URLs. If the path specifier is a URI, then it is checked for a positive response
* before the path is returned. URLs can not be used for directories.</LI>
* <LI>The local filesystem, starting from the current directory of the process.</LI>
* <LI>The class path.</LI>
* </OL>
*
* @param pathspecs A specifier for a URL, a directory with a trailing slash, or a file
* with no trailing slash.
* @return A Path
* @throws RuntimeException if none of the specified paths is found in any of the locations
*/
public static Path findPathIn(String... pathspecs) {
Optional<Path> found = FindOptionalPathIn(pathspecs);
return found.orElseThrow();
}
public static Optional<Path> FindOptionalPathIn(String... pathspecs) {
Path foundPath = null;
for (String pathspec : pathspecs) {
if (isRemote(pathspec)) {
try {
Optional<InputStream> inputStreamForUrl = getInputStreamForUrl(pathspec);
if (inputStreamForUrl.isPresent()) {
foundPath = Path.of(URI.create(pathspec));
logger.debug("Found accessible remote file at " + foundPath.toString());
}
} catch (Exception ignored) {
}
} else {
boolean wantsADirectory = pathspec.endsWith(FileSystems.getDefault().getSeparator());
String candidatePath = wantsADirectory ? pathspec.substring(0, pathspec.length() - 1) : pathspec;
Path candidate = Path.of(candidatePath);
try {
FileSystemProvider provider = candidate.getFileSystem().provider();
provider.checkAccess(candidate, AccessMode.READ);
BasicFileAttributes attrs = provider.readAttributes(candidate, BasicFileAttributes.class);
boolean foundADirectory = attrs.isDirectory();
if (wantsADirectory != foundADirectory) {
throw new RuntimeException("for path " + pathspec + ", user wanted a " +
(wantsADirectory ? "directory" : "file") + ", but found a " +
(foundADirectory ? "directory" : "file") + " while searching paths " +
Arrays.toString(pathspecs));
}
foundPath = candidate;
} catch (Exception ignored) {
}
if (foundPath == null) {
try {
URL url = ClassLoader.getSystemResource(candidatePath);
if (url != null) {
URI uri = URI.create(url.toExternalForm());
foundPath = getPathInFilesystem(uri);
logger.debug("Found path in classpath: " + candidatePath + ": " + foundPath.toString());
}
} catch (Exception e) {
logger.trace("Error while looking in classpath for " + e.getMessage(), e);
}
}
}
}
return Optional.ofNullable(foundPath);
}
public static Optional<InputStream> getInputStreamForUrl(String path) {
URL url;
try {
url = new URL(path);
InputStream inputStream = url.openStream();
if (inputStream != null) {
return Optional.of(inputStream);
}
} catch (Exception e) {
throw new RuntimeException(e);
}
return null;
}
public static CSVParser readDelimFile(String basename, char delimiter, String... searchPaths) {
Reader reader = findRequiredReader(basename, "csv", searchPaths);
CSVFormat format = CSVFormat.newFormat(delimiter).withFirstRecordAsHeader();
try {
CSVParser parser = new CSVParser(reader, format);
return parser;
} catch (IOException e) {
throw new RuntimeException(e);
}
}
public static Reader findRequiredReader(String basename, String extension, String... searchPaths) {
Optional<Reader> optionalReader = findOptionalReader(basename, extension, searchPaths);
return optionalReader.orElseThrow(() -> new RuntimeException(
"Unable to find " + basename + " with extension " + extension + " in file system or in classpath, with"
+ " search paths: " + Arrays.stream(searchPaths).collect(Collectors.joining(","))
));
}
public static List<String> readFileLines(String basename, String... searchPaths) {
InputStream requiredStreamOrFile = findRequiredStreamOrFile(basename, "", DATA_DIR);
try (BufferedReader buffer = new BufferedReader((new InputStreamReader(requiredStreamOrFile)))) {
List<String> collected = buffer.lines().collect(Collectors.toList());
return collected;
} catch (IOException ioe) {
throw new RuntimeException("Error while reading required file to string", ioe);
}
}
public static Optional<Reader> findOptionalReader(String basename, String extenion, String... searchPaths) {
return findOptionalStreamOrFile(basename, extenion, searchPaths)
.map(InputStreamReader::new)
.map(BufferedReader::new);
}
private synchronized static Path getPathInFilesystem(URI uri) {
FileSystem fileSystem = null;
try {
fileSystem = FileSystems.getFileSystem(uri);
} catch (FileSystemNotFoundException ignored) {
try {
fileSystem = FileSystems.newFileSystem(uri, new HashMap<>());
} catch (IOException e) {
throw new RuntimeException(e);
}
}
return Path.of(uri);
}
public static InputStream findRequiredStreamOrFile(String basename, String extension, String... searchPaths) {
Optional<InputStream> optionalStreamOrFile = findOptionalStreamOrFile(basename, extension, searchPaths);
return optionalStreamOrFile.orElseThrow(() -> new RuntimeException(
"Unable to find " + basename + " with extension " + extension + " in file system or in classpath, with"
+ " search paths: " + Arrays.stream(searchPaths).collect(Collectors.joining(","))
));
}
public static Optional<InputStream> findOptionalStreamOrFile(String basename, String extension, String... searchPaths) {
boolean needsExtension = (extension != null && !extension.isEmpty() && !basename.endsWith("." + extension));
String filename = basename + (needsExtension ? "." + extension : "");
ArrayList<String> paths = new ArrayList<String>() {{
add(filename);
if (!isRemote(basename)) {
addAll(Arrays.stream(searchPaths).map(s -> s + File.separator + filename)
.collect(Collectors.toCollection(ArrayList::new)));
}
}};
for (String path : paths) {
Optional<InputStream> stream = getInputStream(path);
if (stream.isPresent()) {
return stream;
}
}
return Optional.empty();
}
/**
* Search for the path
*
* @param basename Basename of path, with or without extension
* @param extension The extension of the filename
* @param searchWithin If enabled, all searchPaths are traversed, looking for a matching suffix pattern.
* @param searchPaths Additional places to look for the path suffix
* @return An optional path
*/
public static Optional<Path> findOptionalPath(String basename, String extension, boolean searchWithin, String... searchPaths) {
boolean needsExtension = (extension != null && !extension.isEmpty() && !basename.endsWith("." + extension));
String filename = basename + (needsExtension ? "." + extension : "");
ArrayList<String> paths = new ArrayList<String>() {{
add(filename);
if (!isRemote(basename)) {
addAll(Arrays.stream(searchPaths).map(s -> s + File.separator + filename)
.collect(Collectors.toCollection(ArrayList::new)));
}
}};
for (String path : paths) {
Optional<InputStream> stream = getInputStream(path);
if (stream.isPresent()) {
return Optional.of(Path.of(path));
}
}
if (searchWithin) {
throw new RuntimeException("not implemented");
// for (String searchPath : searchPaths) {
// NBPathWalker.findEndMatching(Path.of(searchPath), Path.of(filename));
// }
}
return Optional.empty();
}
private static boolean isRemote(String path) {
return (path.toLowerCase().startsWith("http:")
|| path.toLowerCase().startsWith("https:"));
}
public static Optional<InputStream> getInputStream(String path) {
// URLs, if http: or https:
if (isRemote(path)) {
URL url;
try {
url = new URL(path);
InputStream inputStream = url.openStream();
if (inputStream != null) {
return Optional.of(inputStream);
}
} catch (Exception e) {
throw new RuntimeException(e);
}
}
// Files
try {
InputStream stream = new FileInputStream(path);
return Optional.of(stream);
} catch (FileNotFoundException ignored) {
}
// Classpath
ClassLoader classLoader = NBPaths.class.getClassLoader();
InputStream stream = classLoader.getResourceAsStream(path);
if (stream != null) {
return Optional.of(stream);
}
return Optional.empty();
}
public static CSVParser readFileCSV(String basename, String... searchPaths) {
Reader reader = findRequiredReader(basename, "csv", searchPaths);
CSVFormat format = CSVFormat.newFormat(',').withFirstRecordAsHeader();
try {
CSVParser parser = new CSVParser(reader, format);
return parser;
} catch (IOException e) {
throw new RuntimeException(e);
}
}
public static List<String> readDataFileLines(String basename) {
return readFileLines(basename, DATA_DIR);
}
public static String readFile(String basename) {
InputStream requiredStreamOrFile = findRequiredStreamOrFile(basename, "");
try (BufferedReader buffer = new BufferedReader((new InputStreamReader(requiredStreamOrFile)))) {
String filedata = buffer.lines().collect(Collectors.joining("\n"));
return filedata;
} catch (IOException ioe) {
throw new RuntimeException("Error while reading required file to string", ioe);
}
}
public static CharBuffer readDataFileToCharBuffer(String basename) {
return loadFileToCharBuffer(basename, DATA_DIR);
}
public static CharBuffer loadFileToCharBuffer(String filename, String... searchPaths) {
InputStream stream = findRequiredStreamOrFile(filename, "", searchPaths);
CharBuffer linesImage;
try {
InputStreamReader isr = new InputStreamReader(stream);
linesImage = CharBuffer.allocate(1024 * 1024);
while (isr.read(linesImage) > 0) {
}
isr.close();
} catch (IOException e) {
logger.error(e.getMessage());
throw new RuntimeException(e);
}
linesImage.flip();
return linesImage.asReadOnlyBuffer();
}
}

4
virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/core/lfsrs/MetaShift.java
View File

@ -1,6 +1,6 @@
package io.nosqlbench.virtdata.library.basics.core.lfsrs;
import io.nosqlbench.nb.api.pathutil.NBPaths;
import io.nosqlbench.nb.api.content.NBIO;
import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.LogManager;
@ -141,7 +141,7 @@ public class MetaShift {
}
int availableSize= Math.max(registerSize,4);
String maskFileName= String.valueOf(availableSize)+"."+"txt";
List<String> lines = NBPaths.readDataFileLines("lfsrmasks/" + maskFileName);
List<String> lines = NBIO.readLines("lfsrmasks/" + maskFileName);
long[] longs = lines.stream().mapToLong(s -> Long.parseLong(s, 16)).toArray();
return longs;
}

4
virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/distributions/CSVFrequencySampler.java
View File

@ -7,6 +7,7 @@ package io.nosqlbench.virtdata.library.basics.shared.distributions;
*/
import io.nosqlbench.nb.api.content.NBIO;
import io.nosqlbench.virtdata.api.annotations.Categories;
import io.nosqlbench.virtdata.api.annotations.Category;
import io.nosqlbench.virtdata.api.annotations.Example;
@ -14,7 +15,6 @@ import io.nosqlbench.virtdata.api.annotations.ThreadSafeMapper;
import io.nosqlbench.virtdata.library.basics.core.stathelpers.AliasSamplerDoubleInt;
import io.nosqlbench.virtdata.library.basics.core.stathelpers.EvProbD;
import io.nosqlbench.virtdata.library.basics.shared.from_long.to_long.Hash;
import io.nosqlbench.nb.api.pathutil.NBPaths;
import org.apache.commons.csv.CSVParser;
import org.apache.commons.csv.CSVRecord;
import org.apache.commons.math3.stat.Frequency;
@ -72,7 +72,7 @@ public class CSVFrequencySampler implements LongFunction<String> {
Set<String> values = new HashSet<>();
List<EvProbD> frequencies = new ArrayList<>();
CSVParser csvdata = NBPaths.readFileCSV(filename);
CSVParser csvdata = NBIO.readFileCSV(filename);
Frequency freq = new Frequency();
for (CSVRecord csvdatum : csvdata) {
String value = csvdatum.get(columnName);

4
virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/distributions/DelimFrequencySampler.java
View File

@ -7,6 +7,7 @@ package io.nosqlbench.virtdata.library.basics.shared.distributions;
*/
import io.nosqlbench.nb.api.content.NBIO;
import io.nosqlbench.virtdata.api.annotations.Categories;
import io.nosqlbench.virtdata.api.annotations.Category;
import io.nosqlbench.virtdata.api.annotations.Example;
@ -14,7 +15,6 @@ import io.nosqlbench.virtdata.api.annotations.ThreadSafeMapper;
import io.nosqlbench.virtdata.library.basics.core.stathelpers.AliasSamplerDoubleInt;
import io.nosqlbench.virtdata.library.basics.core.stathelpers.EvProbD;
import io.nosqlbench.virtdata.library.basics.shared.from_long.to_long.Hash;
import io.nosqlbench.nb.api.pathutil.NBPaths;
import org.apache.commons.csv.CSVParser;
import org.apache.commons.csv.CSVRecord;
import org.apache.commons.math3.stat.Frequency;
@ -75,7 +75,7 @@ public class DelimFrequencySampler implements LongFunction<String> {
Set<String> values = new HashSet<>();
List<EvProbD> frequencies = new ArrayList<>();
CSVParser csvdata = NBPaths.readDelimFile(filename, delimiter);
CSVParser csvdata = NBIO.readFileDelimCSV(filename,delimiter);
Frequency freq = new Frequency();
for (CSVRecord csvdatum : csvdata) {
String value = csvdatum.get(columnName);

4
virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/distributions/WeightedStringsFromCSV.java
View File

@ -1,12 +1,12 @@
package io.nosqlbench.virtdata.library.basics.shared.distributions;
import io.nosqlbench.nb.api.content.NBIO;
import io.nosqlbench.virtdata.api.annotations.Categories;
import io.nosqlbench.virtdata.api.annotations.Category;
import io.nosqlbench.virtdata.api.annotations.ThreadSafeMapper;
import io.nosqlbench.virtdata.library.basics.core.stathelpers.AliasSamplerDoubleInt;
import io.nosqlbench.virtdata.library.basics.shared.from_long.to_long.Hash;
import io.nosqlbench.virtdata.library.basics.core.stathelpers.EvProbD;
import io.nosqlbench.nb.api.pathutil.NBPaths;
import org.apache.commons.csv.CSVParser;
import org.apache.commons.csv.CSVRecord;
@ -71,7 +71,7 @@ public class WeightedStringsFromCSV implements LongFunction<String> {
this.hash=new Hash();
}
for (String filename: filenames) {
CSVParser csvdata = NBPaths.readFileCSV(filename);
CSVParser csvdata = NBIO.readFileCSV(filename);
for (CSVRecord csvdatum : csvdata) {
String value = csvdatum.get(valueColumn);
values.add(value);

5
virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_int/HashedLineToInt.java
View File

@ -1,5 +1,6 @@
package io.nosqlbench.virtdata.library.basics.shared.from_long.to_int;
import io.nosqlbench.nb.api.content.NBIO;
import io.nosqlbench.virtdata.api.annotations.ThreadSafeMapper;
import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.LogManager;
@ -7,8 +8,6 @@ import org.apache.logging.log4j.LogManager;
import java.util.List;
import java.util.function.LongToIntFunction;
import static io.nosqlbench.nb.api.pathutil.NBPaths.readDataFileLines;
/**
* Return a pseudo-randomly selected integer value from a file of numeric values.
* Each line in the file must contain one parsable integer value.
@ -22,7 +21,7 @@ public class HashedLineToInt implements LongToIntFunction {
public HashedLineToInt(String filename) {
this.filename = filename;
List<String> lines = readDataFileLines(filename);
List<String> lines = NBIO.readLines(filename);
this.values = lines.stream().mapToInt(Integer::parseInt).toArray();
this.intHash = new Hash();
}

4
virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_string/HashedFileExtractToString.java
View File

@ -18,10 +18,10 @@
package io.nosqlbench.virtdata.library.basics.shared.from_long.to_string;
import io.nosqlbench.nb.api.content.NBIO;
import io.nosqlbench.virtdata.api.annotations.Example;
import io.nosqlbench.virtdata.api.annotations.ThreadSafeMapper;
import io.nosqlbench.virtdata.library.basics.shared.from_long.to_int.HashRange;
import io.nosqlbench.nb.api.pathutil.NBPaths;
import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.LogManager;
@ -60,7 +60,7 @@ public class HashedFileExtractToString implements LongFunction<String> {
if (fileDataImage == null) {
synchronized (HashedFileExtractToString.class) {
if (fileDataImage == null) {
CharBuffer image= NBPaths.readDataFileToCharBuffer(fileName);
CharBuffer image= NBIO.readCharBuffer(fileName);
fileDataImage = image;
}
}

5
virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_string/HashedLineToString.java
View File

@ -18,6 +18,7 @@
package io.nosqlbench.virtdata.library.basics.shared.from_long.to_string;
import io.nosqlbench.nb.api.content.NBIO;
import io.nosqlbench.virtdata.api.annotations.ThreadSafeMapper;
import io.nosqlbench.virtdata.library.basics.shared.from_long.to_int.HashRange;
import org.apache.logging.log4j.Logger;
@ -27,8 +28,6 @@ import java.util.ArrayList;
import java.util.List;
import java.util.function.LongFunction;
import static io.nosqlbench.nb.api.pathutil.NBPaths.*;
/**
* Return a pseudo-randomly selected String value from a single line of
* the specified file.
@ -44,7 +43,7 @@ public class HashedLineToString implements LongFunction<String> {
public HashedLineToString(String filename) {
this.filename = filename;
this.lines = readDataFileLines(filename);
this.lines = NBIO.readLines(filename);
this.indexRange = new HashRange(0, lines.size() - 2);
}

4
virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_string/ModuloCSVLineToString.java
View File

@ -18,9 +18,9 @@
package io.nosqlbench.virtdata.library.basics.shared.from_long.to_string;
import io.nosqlbench.nb.api.content.NBIO;
import io.nosqlbench.virtdata.api.annotations.Example;
import io.nosqlbench.virtdata.api.annotations.ThreadSafeMapper;
import io.nosqlbench.nb.api.pathutil.NBPaths;
import org.apache.commons.csv.CSVParser;
import org.apache.commons.csv.CSVRecord;
import org.apache.logging.log4j.Logger;
@ -47,7 +47,7 @@ public class ModuloCSVLineToString implements LongFunction<String> {
@Example({"ModuloCSVLineToString('data/myfile.csv','lat')","load values for 'lat' from the CSV file myfile.csv."})
public ModuloCSVLineToString(String filename, String fieldname) {
this.filename = filename;
CSVParser csvp = NBPaths.readFileCSV(filename);
CSVParser csvp = NBIO.readFileCSV(filename);
Map<String, Integer> headerMap = csvp.getHeaderMap();
if (headerMap==null || headerMap.isEmpty()) {

6
virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_string/ModuloLineToString.java
View File

@ -18,6 +18,7 @@
package io.nosqlbench.virtdata.library.basics.shared.from_long.to_string;
import io.nosqlbench.nb.api.content.NBIO;
import io.nosqlbench.virtdata.api.annotations.ThreadSafeMapper;
import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.LogManager;
@ -26,8 +27,6 @@ import java.util.ArrayList;
import java.util.List;
import java.util.function.LongFunction;
import static io.nosqlbench.nb.api.pathutil.NBPaths.readDataFileLines;
/**
* Select a value from a text file line by modulo division against the number
* of lines in the file.
@ -41,7 +40,8 @@ public class ModuloLineToString implements LongFunction<String> {
public ModuloLineToString(String filename) {
this.filename = filename;
this.lines = readDataFileLines(filename);
this.lines = NBIO.readLines(filename);
}
@Override

4
virtdata-lib-random/src/main/java/io/nosqlbench/virtdata/library/random/RandomFileExtractToString.java
View File

@ -18,8 +18,8 @@
package io.nosqlbench.virtdata.library.random;
import io.nosqlbench.nb.api.content.NBIO;
import io.nosqlbench.virtdata.api.annotations.DeprecatedFunction;
import io.nosqlbench.nb.api.pathutil.NBPaths;
import org.apache.commons.math3.distribution.IntegerDistribution;
import org.apache.commons.math3.distribution.UniformIntegerDistribution;
import org.apache.commons.math3.random.MersenneTwister;
@ -60,7 +60,7 @@ public class RandomFileExtractToString implements LongFunction<String> {
if (fileDataImage == null) {
synchronized (RandomFileExtractToString.class) {
if (fileDataImage == null) {
CharBuffer image= NBPaths.readDataFileToCharBuffer(fileName);
CharBuffer image= NBIO.readCharBuffer(fileName);
fileDataImage = image;
}
}

4
virtdata-lib-random/src/main/java/io/nosqlbench/virtdata/library/random/RandomLineToInt.java
View File

@ -1,7 +1,7 @@
package io.nosqlbench.virtdata.library.random;
import io.nosqlbench.nb.api.content.NBIO;
import io.nosqlbench.virtdata.api.annotations.DeprecatedFunction;
import io.nosqlbench.nb.api.pathutil.NBPaths;
import org.apache.commons.math3.distribution.IntegerDistribution;
import org.apache.commons.math3.distribution.UniformIntegerDistribution;
import org.apache.commons.math3.random.MersenneTwister;
@ -26,7 +26,7 @@ public class RandomLineToInt implements LongToIntFunction {
public RandomLineToInt(String filename, long seed) {
this.filename = filename;
this.lines = NBPaths.readDataFileLines(filename);
this.lines = NBIO.readLines(filename);
this.rng = new MersenneTwister(seed);
this.itemDistribution= new UniformIntegerDistribution(rng, 0, lines.size()-2);
}

9
virtdata-lib-random/src/main/java/io/nosqlbench/virtdata/library/random/RandomLineToString.java
View File

@ -18,8 +18,8 @@
package io.nosqlbench.virtdata.library.random;
import io.nosqlbench.nb.api.content.NBIO;
import io.nosqlbench.virtdata.api.annotations.DeprecatedFunction;
import io.nosqlbench.nb.api.pathutil.NBPaths;
import org.apache.commons.math3.distribution.IntegerDistribution;
import org.apache.commons.math3.distribution.UniformIntegerDistribution;
import org.apache.commons.math3.random.MersenneTwister;
@ -45,22 +45,21 @@ public class RandomLineToString implements LongFunction<String> {
public RandomLineToString(String filename) {
this.rng = new MersenneTwister(System.nanoTime());
this.filename = filename;
this.lines = NBPaths.readDataFileLines(filename);
this.lines = NBIO.readLines(filename);
itemDistribution= new UniformIntegerDistribution(rng, 0, lines.size()-2);
}
public RandomLineToString(String filename, MersenneTwister rng) {
this.rng = rng;
this.filename = filename;
this.lines = NBPaths.readDataFileLines(filename);
this.lines = NBPaths.readDataFileLines(filename);
this.lines = NBIO.readLines(filename);
itemDistribution= new UniformIntegerDistribution(rng, 0, lines.size()-2);
}
public RandomLineToString(String filename, long seed) {
this.rng = new MersenneTwister(seed);
this.filename = filename;
this.lines = NBPaths.readDataFileLines(filename);
this.lines = NBIO.readLines(filename);
itemDistribution= new UniformIntegerDistribution(rng, 0, lines.size()-2);
}

4
virtdata-userlibs/src/main/java/io/nosqlbench/virtdata/userlibs/apps/docsapp/AutoDocsApp.java
View File

@ -2,9 +2,9 @@ package io.nosqlbench.virtdata.userlibs.apps.docsapp;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import io.nosqlbench.nb.api.content.NBIO;
import io.nosqlbench.virtdata.api.annotations.Category;
import io.nosqlbench.virtdata.core.bindings.VirtDataDocs;
import io.nosqlbench.nb.api.pathutil.NBPaths;
import io.nosqlbench.virtdata.api.processors.DocFuncData;
import io.nosqlbench.virtdata.userlibs.apps.docsapp.fdocs.FDoc;
import io.nosqlbench.virtdata.userlibs.apps.docsapp.fdocs.FDocCat;
@ -143,7 +143,7 @@ public class AutoDocsApp implements Runnable {
String[] blurbsdirs = blurbsDirs.split(":");
for (String blurbsdir : blurbsdirs) {
Optional<Path> bdir = NBPaths.FindOptionalPathIn(blurbsdir+"/");
Optional<Path> bdir = NBIO.findFirstLocalPath(blurbsdir+"/");
if (bdir.isPresent()) {
Path blurbsFile = bdir.get().resolve(Path.of(outputname).getFileName().toString());
if (Files.exists(blurbsFile)) {