IntenseFinance/nosqlbench
3
0
Fork 0
mirror of https://github.com/nosqlbench/nosqlbench.git synced 2025-02-25 18:55:28 -06:00
Code Issues Packages Projects Releases Wiki Activity

nosqlbench-1071 NBIO should favor finding exactly specified file path as a short circuit

Browse Source
This commit is contained in:
Jonathan Shook 2023-02-10 10:20:45 -06:00
parent 756e1e5ff0
commit 70408621dd
29 changed files with 555 additions and 250 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/Cqld4Space.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -99,7 +99,7 @@ public class Cqld4Space implements AutoCloseable {
int port = cfg.getOrDefault("port", 9042);
Optional<String> scb = cfg.getOptional(String.class, "secureconnectbundle", "scb");
scb.flatMap(s -> NBIO.all().name(s).first().map(Content::getInputStream))
scb.flatMap(s -> NBIO.all().pathname(s).first().map(Content::getInputStream))
.map(builder::withCloudSecureConnectBundle);
Optional<List<InetSocketAddress>> contactPointsOption = cfg
@ -229,14 +229,14 @@ public class Cqld4Space implements AutoCloseable {
for (String loaderspec : loaderspecs) {
// path
Optional<Content<?>> fsconfig = NBIO.fs().name(driverconfig).first();
Optional<Content<?>> fsconfig = NBIO.fs().pathname(driverconfig).first();
if (fsconfig.isPresent()) {
loaders.add(DriverConfigLoader.fromPath(fsconfig.get().asPath()));
continue;
}
// classpath
Optional<Content<?>> cpconfig = NBIO.classpath().name(driverconfig).first();
Optional<Content<?>> cpconfig = NBIO.classpath().pathname(driverconfig).first();
if (cpconfig.isPresent()) {
loaders.add(DriverConfigLoader.fromClasspath(driverconfig));
continue;
@ -244,7 +244,7 @@ public class Cqld4Space implements AutoCloseable {
// URLs
try {
Optional<Content<?>> removeconf = NBIO.remote().name(driverconfig).first();
Optional<Content<?>> removeconf = NBIO.remote().pathname(driverconfig).first();
if (removeconf.isPresent()) {
loaders.add(DriverConfigLoader.fromUrl(removeconf.get().getURI().toURL()));
continue;

4
adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/core/CGDefaultCqlBindings.java
View File

@ -37,9 +37,9 @@ public class CGDefaultCqlBindings implements BindingsLibrary {
public CGDefaultCqlBindings() {
Content<?> content = NBIO.all()
.name(DEFAULT_BINDINGS_FILE)
.pathname(DEFAULT_BINDINGS_FILE)
.first()
.or(() -> NBIO.all().prefix(DEFAULT_CFG_DIR).name(DEFAULT_BINDINGS_FILE).first())
.or(() -> NBIO.all().searchPrefixes(DEFAULT_CFG_DIR).pathname(DEFAULT_BINDINGS_FILE).first())
.orElseThrow(
() -> new RuntimeException("Unable to load " + DEFAULT_BINDINGS_FILE +
", from local dir or internally as cqlgen" + DEFAULT_BINDINGS_FILE)

4
adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/core/CGWorkloadExporter.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -125,7 +125,7 @@ public class CGWorkloadExporter implements BundledApp {
Yaml yaml = new Yaml();
CGWorkloadExporter exporter;
Content<?> cqlgencfg = NBIO.local().prefix("cqlgen").name("cqlgen").extension("conf").first().orElseThrow();
Content<?> cqlgencfg = NBIO.local().searchPrefixes("cqlgen").pathname("cqlgen").extensionSet("conf").first().orElseThrow();
if (cqlgencfg == null) {
throw new RuntimeException("Unable to load cqlgen.conf");
}

4
adapter-http/src/main/java/io/nosqlbench/adapter/http/statuscodes/HttpStatusCodes.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -30,7 +30,7 @@ public class HttpStatusCodes {
private static final IetfStatusCode[] codes = loadMap();
private static IetfStatusCode[] loadMap() {
Content<?> csv = NBIO.local().name("ietf-http-status-codes").extension("csv").one();
Content<?> csv = NBIO.local().pathname("ietf-http-status-codes").extensionSet("csv").one();
InputStreamReader isr = new InputStreamReader(csv.getInputStream());
IetfStatusCode[] codes = new IetfStatusCode[600];

6
adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/OpsLoader.java
View File

@ -54,7 +54,7 @@ public class OpsLoader {
public static OpsDocList loadPath(String path, Map<String, ?> params, String... searchPaths) {
String[] extensions = path.indexOf('.')>-1 ? new String[]{} : YAML_EXTENSIONS;
Content<?> foundPath = NBIO.all().prefix(searchPaths).name(path).extension(extensions).first()
Content<?> foundPath = NBIO.all().searchPrefixes(searchPaths).pathname(path).extensionSet(extensions).first()
.orElseThrow(() -> new RuntimeException("Unable to load path '" + path + "'"));
OpTemplateFormat fmt = OpTemplateFormat.valueOfURI(foundPath.getURI());
return loadString(foundPath.asString(), fmt, params, foundPath.getURI());
@ -81,9 +81,7 @@ public class OpsLoader {
transformer.checkpointAccesses().forEach((k, v) -> {
layered.addTemplateVariable(k, v);
if (params.containsKey(k)) {
params.remove(k);
}
params.remove(k);
});
return layered;

2
adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawOpsLoader.java
View File

@ -69,7 +69,7 @@ public class RawOpsLoader {
String data = null;
try {
Optional<Content<?>> oyaml = NBIO.all().prefix(searchPaths).name(path).extension(YAML_EXTENSIONS).first();
Optional<Content<?>> oyaml = NBIO.all().searchPrefixes(searchPaths).pathname(path).extensionSet(YAML_EXTENSIONS).first();
data = oyaml.map(Content::asString).orElseThrow(() -> new BasicError("Unable to load " + path));
return loadString(data);
} catch (Exception e) {

2
adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawYamlLoader.java
View File

@ -65,7 +65,7 @@ public class RawYamlLoader {
String data = null;
try {
Optional<Content<?>> oyaml = NBIO.all().prefix(searchPaths).name(path).extension(YAML_EXTENSIONS).first();
Optional<Content<?>> oyaml = NBIO.all().searchPrefixes(searchPaths).pathname(path).extensionSet(YAML_EXTENSIONS).first();
data = oyaml.map(Content::asString).orElseThrow(() -> new BasicError("Unable to load " + path));
return loadString(logger, data);
} catch (Exception e) {

10
adapters-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/DriverAdapter.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -28,15 +28,11 @@ import io.nosqlbench.api.config.standard.NBConfiguration;
import io.nosqlbench.api.content.Content;
import io.nosqlbench.api.content.NBIO;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.function.Function;
import java.util.function.LongFunction;
import java.util.stream.Stream;
/**
* <P>The DriverAdapter interface is expected to be the replacement
@ -185,10 +181,10 @@ public interface DriverAdapter<OPTYPE extends Op, SPACETYPE> {
String dev_docspath = "adapter-" + this.getAdapterName() + "/src/main/resources/docs/" + this.getAdapterName();
String cp_docspath = "docs/" + this.getAdapterName();
Optional<Content<?>> bundled_docs = NBIO.local().name(dev_docspath, cp_docspath).first();
Optional<Content<?>> bundled_docs = NBIO.local().pathname(dev_docspath, cp_docspath).first();
bundled_docs.map(Content::asPath).ifPresent(docs::addContentsOf);
Optional<Content<?>> maindoc = NBIO.local().name("/src/main/resources/" + this.getAdapterName() + ".md", this.getAdapterName() + ".md").first();
Optional<Content<?>> maindoc = NBIO.local().pathname("/src/main/resources/" + this.getAdapterName() + ".md", this.getAdapterName() + ".md").first();
maindoc.map(Content::asPath).ifPresent(docs::addPath);

2
adapters-api/src/main/java/io/nosqlbench/engine/api/specifications/SpecDocsManifest.java
View File

@ -30,7 +30,7 @@ public class SpecDocsManifest implements BundledMarkdownManifest {
@Override
public DocsBinder getDocs() {
Docs docs = new Docs().namespace("workload_definition");
List<Content<?>> specfiles = NBIO.classpath().prefix("workload_definition/").extension(".md").list();
List<Content<?>> specfiles = NBIO.classpath().searchPrefixes("workload_definition/").extensionSet(".md").list();
for (Content<?> specfile : specfiles) {
docs.addPath(specfile.asPath());
}

46
engine-api/src/main/java/io/nosqlbench/engine/api/scenarios/NBCLIScenarioParser.java
View File

@ -50,10 +50,10 @@ public class NBCLIScenarioParser {
public static boolean isFoundWorkload(String workload, String... includes) {
Optional<Content<?>> found = NBIO.all()
.prefix("activities")
.prefix(includes)
.name(workload)
.extension(RawOpsLoader.YAML_EXTENSIONS)
.searchPrefixes("activities")
.searchPrefixes(includes)
.pathname(workload)
.extensionSet(RawOpsLoader.YAML_EXTENSIONS)
.first();
return found.isPresent();
}
@ -64,10 +64,10 @@ public class NBCLIScenarioParser {
String workloadName = arglist.removeFirst();
Optional<Content<?>> found = NBIO.all()
.prefix("activities")
.prefix(includes)
.name(workloadName)
.extension(RawOpsLoader.YAML_EXTENSIONS)
.searchPrefixes("activities")
.searchPrefixes(includes)
.pathname(workloadName)
.extensionSet(RawOpsLoader.YAML_EXTENSIONS)
.first();
//
Content<?> workloadContent = found.orElseThrow();
@ -108,10 +108,10 @@ public class NBCLIScenarioParser {
// Load in named scenario
Content<?> yamlWithNamedScenarios = NBIO.all()
.prefix(SEARCH_IN)
.prefix(includes)
.name(workloadName)
.extension(RawOpsLoader.YAML_EXTENSIONS)
.searchPrefixes(SEARCH_IN)
.searchPrefixes(includes)
.pathname(workloadName)
.extensionSet(RawOpsLoader.YAML_EXTENSIONS)
.first().orElseThrow();
// TODO: The yaml needs to be parsed with arguments from each command independently to support template vars
OpsDocList scenariosYaml = OpsLoader.loadContent(yamlWithNamedScenarios, new LinkedHashMap<>(userProvidedParams));
@ -320,8 +320,8 @@ public class NBCLIScenarioParser {
}
}
Content<?> content = NBIO.all().prefix(SEARCH_IN)
.name(referenced).extension(RawOpsLoader.YAML_EXTENSIONS)
Content<?> content = NBIO.all().searchPrefixes(SEARCH_IN)
.pathname(referenced).extensionSet(RawOpsLoader.YAML_EXTENSIONS)
.one();
OpsDocList stmts = null;
@ -379,14 +379,14 @@ public class NBCLIScenarioParser {
public static List<WorkloadDesc> getWorkloadsWithScenarioScripts(boolean defaultIncludes, String... includes) {
NBPathsAPI.GetPrefix searchin = NBIO.all();
NBPathsAPI.GetPrefixes searchin = NBIO.all();
if (defaultIncludes) {
searchin = searchin.prefix(SEARCH_IN);
searchin = searchin.searchPrefixes(SEARCH_IN);
}
List<Content<?>> activities = searchin
.prefix(includes)
.extension(RawOpsLoader.YAML_EXTENSIONS)
.searchPrefixes(includes)
.extensionSet(RawOpsLoader.YAML_EXTENSIONS)
.list();
return filterForScenarios(activities);
@ -395,15 +395,15 @@ public class NBCLIScenarioParser {
public static List<String> getScripts(boolean defaultIncludes, String... includes) {
NBPathsAPI.GetPrefix searchin = NBIO.all();
NBPathsAPI.GetPrefixes searchin = NBIO.all();
if (defaultIncludes) {
searchin = searchin.prefix(SEARCH_IN);
searchin = searchin.searchPrefixes(SEARCH_IN);
}
List<Path> scriptPaths = searchin
.prefix("scripts/auto")
.prefix(includes)
.extension("js")
.searchPrefixes("scripts/auto")
.searchPrefixes(includes)
.extensionSet("js")
.list().stream().map(Content::asPath).collect(Collectors.toList());
List<String> scriptNames = new ArrayList();

2
engine-cli/src/main/java/io/nosqlbench/engine/cli/BasicScriptBuffer.java
View File

@ -145,7 +145,7 @@ public class BasicScriptBuffer implements ScriptBuffer {
logger.debug(() -> "Looking for " + script_path);
Content<?> one = NBIO.all().prefix("scripts").name(script_path).extension("js").one();
Content<?> one = NBIO.all().searchPrefixes("scripts").pathname(script_path).extensionSet("js").one();
scriptData = one.asString();
StrInterpolator interpolator = new StrInterpolator(cmd.getParams());

12
engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLI.java
View File

@ -306,16 +306,16 @@ public class NBCLI implements Function<String[], Integer> {
logger.debug(() -> "user requests to copy out " + resourceToCopy);
Optional<Content<?>> tocopy = NBIO.classpath()
.prefix("activities")
.prefix(options.wantsIncludes())
.name(resourceToCopy).extension(RawOpsLoader.YAML_EXTENSIONS).first();
.searchPrefixes("activities")
.searchPrefixes(options.wantsIncludes())
.pathname(resourceToCopy).extensionSet(RawOpsLoader.YAML_EXTENSIONS).first();
if (tocopy.isEmpty()) {
tocopy = NBIO.classpath()
.prefix().prefix(options.wantsIncludes())
.prefix(options.wantsIncludes())
.name(resourceToCopy).first();
.searchPrefixes().searchPrefixes(options.wantsIncludes())
.searchPrefixes(options.wantsIncludes())
.pathname(resourceToCopy).first();
}
Content<?> data = tocopy.orElseThrow(

6
engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLICommandParser.java
View File

@ -71,9 +71,9 @@ public class NBCLICommandParser {
break;
default:
Optional<Content<?>> scriptfile = NBIO.local()
.prefix("scripts/auto")
.name(word)
.extension("js")
.searchPrefixes("scripts/auto")
.pathname(word)
.extensionSet("js")
.first();
//Script

10
engine-cli/src/main/java/io/nosqlbench/engine/cli/PathCanonicalizer.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -35,9 +35,9 @@ public class PathCanonicalizer {
public String canonicalizePath(String path) {
Optional<Content<?>> found = NBIO.local().prefix("activities")
.prefix(includes)
.name(path)
Optional<Content<?>> found = NBIO.local().searchPrefixes("activities")
.searchPrefixes(includes)
.pathname(path)
.first();
if (found.isPresent()) {
@ -46,7 +46,7 @@ public class PathCanonicalizer {
rewriteTo=(rewriteTo.startsWith(separator) ? rewriteTo.substring(1) : rewriteTo);
if (!rewriteTo.equals(path)) {
if (NBIO.local().prefix("activities").prefix(includes).name(rewriteTo).first().isPresent()) {
if (NBIO.local().searchPrefixes("activities").searchPrefixes(includes).pathname(rewriteTo).first().isPresent()) {
logger.info("rewrote path for " + path + " as " + rewriteTo);
return rewriteTo;
} else {

4
engine-clients/src/test/java/io/nosqlbench/engine/clients/prometheus/PMatrixElemTest.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -33,7 +33,7 @@ public class PMatrixElemTest {
@Disabled
public void testMatrixElem() {
Gson gson = new GsonBuilder().create();
String json = NBIO.classpath().name("test.json").one().asString();
String json = NBIO.classpath().pathname("test.json").one().asString();
Type type = new TypeToken<PromQueryResult<PMatrixData>>() {
}.getType();
Object result = gson.fromJson(json, type);

6
engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/activity/ActivityTypeLoader.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -114,8 +114,8 @@ public class ActivityTypeLoader {
private Set<URL> addLibDir(Set<URL> urlsToAdd, Path libpath) {
Set<URL> urls = NBIO.local()
.prefix(libpath.toString())
.extension(".jar")
.searchPrefixes(libpath.toString())
.extensionSet(".jar")
.list().stream().map(Content::getURL)
.collect(Collectors.toSet());
urlsToAdd.addAll(urls);

10
engine-core/src/main/java/io/nosqlbench/engine/core/metadata/MarkdownFinder.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -53,10 +53,10 @@ public class MarkdownFinder {
public String forResourceMarkdown(String s, String... additionalSearchPaths) {
Optional<Content<?>> docs = NBIO.local()
.prefix("docs")
.prefix(additionalSearchPaths)
.name(s)
.extension(".md")
.searchPrefixes("docs")
.searchPrefixes(additionalSearchPaths)
.pathname(s)
.extensionSet(".md")
.first();
return docs.map(Content::asString).orElse(null);

24
engine-docker/src/main/java/io/nosqlbench/engine/docker/DockerMetricsManager.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -82,19 +82,19 @@ public class DockerMetricsManager {
String GRAFANA_IMG = "grafana/grafana";
tag = (tag == null || tag.isEmpty()) ? "latest" : tag;
String name = "grafana";
List<Integer> port = Arrays.asList(3000);
List<Integer> port = List.of(3000);
boolean grafanaFilesExist = grafanaFilesExist();
if (!grafanaFilesExist) {
setupGrafanaFiles(ip);
}
List<String> volumeDescList = Arrays.asList(
userHome + "/.nosqlbench/grafana:/var/lib/grafana:rw"
//cwd+"/docker-metrics/grafana:/grafana",
//cwd+"/docker-metrics/grafana/datasources:/etc/grafana/provisioning/datasources",
//cwd+"/docker-metrics/grafana/dashboardconf:/etc/grafana/provisioning/dashboards"
//,cwd+"/docker-metrics/grafana/dashboards:/var/lib/grafana/dashboards:ro"
List<String> volumeDescList = List.of(
userHome + "/.nosqlbench/grafana:/var/lib/grafana:rw"
//cwd+"/docker-metrics/grafana:/grafana",
//cwd+"/docker-metrics/grafana/datasources:/etc/grafana/provisioning/datasources",
//cwd+"/docker-metrics/grafana/dashboardconf:/etc/grafana/provisioning/dashboards"
//,cwd+"/docker-metrics/grafana/dashboards:/var/lib/grafana/dashboards:ro"
);
List<String> envList = Arrays.asList(
"GF_SECURITY_ADMIN_PASSWORD=admin",
@ -125,7 +125,7 @@ public class DockerMetricsManager {
logger.info("preparing to start docker metrics");
String PROMETHEUS_IMG = "prom/prometheus";
String name = "prom";
List<Integer> port = Arrays.asList(9090);
List<Integer> port = List.of(9090);
if (!promFilesExist()) {
setupPromFiles(ip);
@ -175,7 +175,7 @@ public class DockerMetricsManager {
setupGraphiteFiles(volumeDescList);
List<String> envList = Arrays.asList();
List<String> envList = List.of();
String reload = null;
List<String> linkNames = new ArrayList();
@ -342,7 +342,7 @@ public class DockerMetricsManager {
private void configureGrafana() {
List<Content<?>> dashboardContent = NBIO.all().prefix("docker/dashboards").extension(".json").list();
List<Content<?>> dashboardContent = NBIO.all().searchPrefixes("docker/dashboards").extensionSet(".json").list();
for (Content<?> content : dashboardContent) {
String dashboardData = content.asString();
@ -355,7 +355,7 @@ public class DockerMetricsManager {
}
List<Content<?>> datasources = NBIO.all().prefix("docker/datasources").extension(".yaml").list();
List<Content<?>> datasources = NBIO.all().searchPrefixes("docker/datasources").extensionSet(".yaml").list();
for (Content<?> datasource : datasources) {
String datasourceContent = datasource.asString();

4
nb-api/src/main/java/io/nosqlbench/api/apps/BundledApp.java
View File

@ -38,10 +38,10 @@ public interface BundledApp extends ToIntFunction<String[]> {
String dev_docspath = "app-" + this.getBundledAppName() + "/src/main/resources/docs/" + this.getBundledAppName();
String cp_docspath = "docs/" + this.getBundledAppName();
Optional<Content<?>> bundled_docs = NBIO.local().name(dev_docspath, cp_docspath).first();
Optional<Content<?>> bundled_docs = NBIO.local().pathname(dev_docspath, cp_docspath).first();
bundled_docs.map(Content::asPath).ifPresent(docs::addContentsOf);
Optional<Content<?>> maindoc = NBIO.local().name("/src/main/resources/" + this.getBundledAppName() + ".md", this.getBundledAppName() + ".md").first();
Optional<Content<?>> maindoc = NBIO.local().pathname("/src/main/resources/" + this.getBundledAppName() + ".md", this.getBundledAppName() + ".md").first();
maindoc.map(Content::asPath).ifPresent(docs::addPath);

4
nb-api/src/main/java/io/nosqlbench/api/config/params/DataSources.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -45,7 +45,7 @@ public class DataSources {
String filename = data.substring("IMPORT{".length(), data.length() - 1);
Path filepath = Path.of(filename);
src = NBIO.all().name(filename).first()
src = NBIO.all().pathname(filename).first()
.map(c -> {
logger.debug(() -> "found 'data' at " + c.getURI());
return c.asString();

7
nb-api/src/main/java/io/nosqlbench/api/config/standard/ConfigLoader.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -103,7 +103,7 @@ public class ConfigLoader {
String filename = data.substring("IMPORT{".length(), data.length() - 1);
Path filepath = Path.of(filename);
data = NBIO.all().name(filename).first()
data = NBIO.all().pathname(filename).first()
.map(c -> {
logger.debug(() -> "found 'data' at " + c.getURI());
return c.asString();
@ -111,9 +111,8 @@ public class ConfigLoader {
}
if (data.startsWith("{") || data.startsWith("[")) {
JsonParser parser = new JsonParser();
JsonElement jsonElement = parser.parse(data);
JsonElement jsonElement = JsonParser.parseString(data);
if (jsonElement.isJsonArray()) {
JsonArray asJsonArray = jsonElement.getAsJsonArray();
for (JsonElement element : asJsonArray) {

297
nb-api/src/main/java/io/nosqlbench/api/content/NBIO.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -31,15 +31,13 @@ import java.util.regex.Pattern;
import java.util.stream.Collectors;
/**
* NBIO is a helper utility packaged as a search builder and fluent API.
* It uses value semantics internally, so it is safe to re-use as a
* stateful configuration builder for finding files in various ways.
* NBIO is a helper utility packaged as a search builder and fluent API. It uses value semantics internally, so it is
* safe to re-use as a stateful configuration builder for finding files in various ways.
* <p>
* Since this is meant to ease development around a usually over-complicated
* surface area in the JVM (Files, Paths, URIs, accessing data, knowing where it comes
* from, searching for it, etc), more emphasis was put on ease of use and
* clarity than efficiency. This set of classes is not expected to be used
* much in NoSqlBench after workload initialization, so is not performance oriented
* Since this is meant to ease development around a usually over-complicated surface area in the JVM (Files, Paths,
* URIs, accessing data, knowing where it comes from, searching for it, etc), more emphasis was put on ease of use and
* clarity than efficiency. This set of classes is not expected to be used much in NoSqlBench after workload
* initialization, so is not performance oriented.
*/
public class NBIO implements NBPathsAPI.Facets {
@ -52,7 +50,8 @@ public class NBIO implements NBPathsAPI.Facets {
private URIResolver resolver;
private List<String> names = new ArrayList<>();
private List<String> extensions = new ArrayList<>();
// private List<String> extensions = new ArrayList<>();
private List<Set<String>> extensionSets = new ArrayList<>();
private Set<String> prefixes = new HashSet<>(Arrays.asList(globalIncludes));
private NBIO() {
@ -61,15 +60,15 @@ public class NBIO implements NBPathsAPI.Facets {
private NBIO(URIResolver resolver,
Set<String> prefixes,
List<String> names,
List<String> extensions) {
List<Set<String>> extensionSets) {
this.resolver = resolver;
this.prefixes = prefixes;
this.names = names;
this.extensions = extensions;
this.extensionSets = extensionSets;
}
public static List<String> readLines(String filename) {
Content<?> data = NBIO.all().prefix("data").name(filename).first().orElseThrow(
Content<?> data = NBIO.all().searchPrefixes("data").pathname(filename).first().orElseThrow(
() -> new BasicError("Unable to read lines from " + filename)
);
String[] split = data.getCharBuffer().toString().split("\n");
@ -94,26 +93,26 @@ public class NBIO implements NBPathsAPI.Facets {
private static InputStream readInputStream(String filename, String... searchPaths) {
return NBIO.all().prefix(searchPaths).name(filename).one().getInputStream();
return NBIO.all().searchPrefixes(searchPaths).pathname(filename).one().getInputStream();
}
private static Reader readReader(String filename, String... searchPaths) {
return NBIO.all().prefix(searchPaths).name(filename).one().getReader();
return NBIO.all().searchPrefixes(searchPaths).pathname(filename).one().getReader();
}
public static CharBuffer readCharBuffer(String fileName, String... searchPaths) {
return NBIO.all().prefix(searchPaths).name(fileName).one().getCharBuffer();
return NBIO.all().searchPrefixes(searchPaths).pathname(fileName).one().getCharBuffer();
}
public static Path getFirstLocalPath(String... potentials) {
Optional<Content<?>> first = NBIO.local().name(potentials).first();
Optional<Content<?>> first = NBIO.local().pathname(potentials).first();
return first.orElseThrow(
() -> new BasicError("Unable to find loadable content at " + String.join(",", potentials))
).asPath();
}
public static Optional<Path> findFirstLocalPath(String... potentials) {
Optional<Content<?>> first = NBIO.local().name(potentials).first();
Optional<Content<?>> first = NBIO.local().pathname(potentials).first();
Optional<Path> path = first.map(Content::asPath);
return path;
}
@ -123,46 +122,67 @@ public class NBIO implements NBPathsAPI.Facets {
}
/**
* {@inheritDoc}
*/
@Override
public NBPathsAPI.GetPrefix localContent() {
public NBPathsAPI.GetPrefixes localContent() {
this.resolver = URIResolvers.inFS().inCP();
return this;
}
/**
* {@inheritDoc}
*/
@Override
public NBPathsAPI.GetPrefix remoteContent() {
public NBPathsAPI.GetPrefixes remoteContent() {
this.resolver = URIResolvers.inURLs();
return this;
}
/**
* {@inheritDoc}
*/
@Override
public NBPathsAPI.GetPrefix internalContent() {
public NBPathsAPI.GetPrefixes internalContent() {
this.resolver = URIResolvers.inClasspath();
return this;
}
/**
* {@inheritDoc}
*/
@Override
public NBPathsAPI.GetPrefix fileContent() {
public NBPathsAPI.GetPrefixes fileContent() {
this.resolver = URIResolvers.inFS();
return this;
}
/**
* {@inheritDoc}
*/
@Override
public NBPathsAPI.GetPrefix allContent() {
public NBPathsAPI.GetPrefixes allContent() {
this.resolver = URIResolvers.inFS().inCP().inURLs();
return this;
}
/**
* {@inheritDoc}
*/
@Override
public NBPathsAPI.GetPrefix prefix(String... searchPaths) {
public NBPathsAPI.GetPrefixes searchPrefixes(String... searchPaths) {
HashSet<String> addingPaths = new HashSet<>(this.prefixes);
addingPaths.addAll(Arrays.asList(searchPaths));
return new NBIO(resolver, addingPaths, names, extensions);
return new NBIO(resolver, addingPaths, names, extensionSets);
}
private final static Pattern extensionPattern = Pattern.compile("\\.[a-zA-Z]+");
private final static Pattern wildcardsPattern = Pattern.compile(".*?[^?+*][\\?\\+\\*].*");
/**
* {@inheritDoc}
*/
@Override
public NBPathsAPI.DoSearch search(String... searches) {
List<String> prefixesToAdd = new ArrayList<>();
@ -185,7 +205,7 @@ public class NBIO implements NBPathsAPI.Facets {
int literalsTill = 0;
while (true) {
if (literalsTill>= parts.length-1) {
if (literalsTill >= parts.length - 1) {
break;
}
if (wildcardsPattern.matcher(parts[literalsTill]).matches()) {
@ -201,31 +221,81 @@ public class NBIO implements NBPathsAPI.Facets {
prefixesToAdd.add(prefix);
String[] nameary = new String[parts.length - literalsTill];
System.arraycopy(parts,literalsTill,nameary,0,nameary.length);
System.arraycopy(parts, literalsTill, nameary, 0, nameary.length);
String name = String.join(File.separator, nameary);
namesToAdd.add(name);
}
return prefix(prefixesToAdd.toArray(new String[]{}))
.name(namesToAdd.toArray(new String[]{}))
.extension(extensionsToAdd.toArray(new String[]{}));
return searchPrefixes(prefixesToAdd.toArray(new String[]{}))
.pathname(namesToAdd.toArray(new String[]{}))
.extensionSet(extensionsToAdd.toArray(new String[]{}));
}
/**
* {@inheritDoc}
*/
@Override
public NBPathsAPI.GetExtension name(String... searchNames) {
public NBPathsAPI.GetExtensions pathname(String... searchNames) {
ArrayList<String> addingNames = new ArrayList<>(this.names);
addingNames.addAll(Arrays.asList(searchNames));
return new NBIO(resolver, prefixes, addingNames, extensions);
return new NBIO(resolver, prefixes, addingNames, extensionSets);
}
/**
* {@inheritDoc}
*/
@Override
public NBPathsAPI.DoSearch extension(String... extensions) {
ArrayList<String> addingExtensions = new ArrayList<>(this.extensions);
for (String addingExtension : extensions) {
addingExtensions.add(dotExtension(addingExtension));
public NBPathsAPI.GetExtensions extensionSet(String... extensions) {
if (extensions.length==0) {
return this;
}
return new NBIO(resolver, prefixes, names, addingExtensions);
return this.extensionSets(
new HashSet<>(
Arrays.asList(extensions).stream()
.map(String::toLowerCase)
.map(s -> s.startsWith(".") ? s : "." + s)
.collect(Collectors.toList())
)
);
}
/**
* {@inheritDoc}
*/
@Override
public NBPathsAPI.GetExtensions extensionSets(Set<String>... cosets) {
for (Set<String> coset : cosets) {
if (coset.size()==0) {
throw new RuntimeException("Extension co-set can not be empty.");
}
for (String entry : coset) {
String adjusted = (entry.startsWith(".") ? entry : "." + entry).toLowerCase();
if (coset.add(adjusted)) {
coset.remove(entry);
}
}
Set<String> addTo = null;
for (Set<String> extensionSet : this.extensionSets) {
Set<String> union = new LinkedHashSet<>(coset);
for (String entry : coset) {
if (extensionSet.contains(entry)) {
addTo = extensionSet;
break;
}
}
if (addTo != null) {
break;
}
}
if (addTo == null) {
addTo = new LinkedHashSet<>();
extensionSets.add(addTo);
}
addTo.addAll(coset);
}
return this;
}
/**
@ -233,7 +303,7 @@ public class NBIO implements NBPathsAPI.Facets {
*
* @return a builder
*/
public static NBPathsAPI.GetPrefix all() {
public static NBPathsAPI.GetPrefixes all() {
return new NBIO().allContent();
}
@ -242,7 +312,7 @@ public class NBIO implements NBPathsAPI.Facets {
*
* @return a builder
*/
public static NBPathsAPI.GetPrefix classpath() {
public static NBPathsAPI.GetPrefixes classpath() {
return new NBIO().internalContent();
}
@ -251,7 +321,7 @@ public class NBIO implements NBPathsAPI.Facets {
*
* @return a builder
*/
public static NBPathsAPI.GetPrefix fs() {
public static NBPathsAPI.GetPrefixes fs() {
return new NBIO().fileContent();
}
@ -260,7 +330,7 @@ public class NBIO implements NBPathsAPI.Facets {
*
* @return a builder
*/
public static NBPathsAPI.GetPrefix local() {
public static NBPathsAPI.GetPrefixes local() {
return new NBIO().localContent();
}
@ -269,11 +339,14 @@ public class NBIO implements NBPathsAPI.Facets {
*
* @return a builder
*/
public static NBPathsAPI.GetPrefix remote() {
public static NBPathsAPI.GetPrefixes remote() {
return new NBIO().remoteContent();
}
/**
* {@inheritDoc}
*/
@Override
public Optional<Content<?>> first() {
@ -296,9 +369,38 @@ public class NBIO implements NBPathsAPI.Facets {
throw new RuntimeException("Invalid code, go fix it, this should never happen.");
}
/**
* {@inheritDoc}
*/
@Override
public Content<?> one() {
if (extensionSets.size()==0) {
for (String name : names) {
Content<?> found = resolver.resolveOne(name);
if (found != null) {
return found;
}
}
} else {
for (String name : names) {
Content<?> found = resolver.resolveOne(name);
if (found != null) {
return found;
}
for (Set<String> extensionSet : extensionSets) {
for (String es : extensionSet) {
for (String extension : extensionSet) {
found = resolver.resolveOne(name + extension);
if (found != null) {
return found;
}
}
}
}
}
}
List<Content<?>> list = list();
if (list.size() == 0) {
@ -313,11 +415,14 @@ public class NBIO implements NBPathsAPI.Facets {
}
/**
* {@inheritDoc}
*/
@Override
public List<List<Content<?>>> resolveEach() {
List<List<Content<?>>> resolved = new ArrayList<>();
for (String name : names) {
LinkedHashSet<String> slotSearchPaths = expandNamesAndSuffixes(List.of(name), new HashSet<>(extensions));
LinkedHashSet<String> slotSearchPaths = expandNamesAndSuffixes(this.prefixes, List.of(name), extensionSets);
Content<?> content = null;
for (String slotSearchPath : slotSearchPaths) {
List<Content<?>> contents = resolver.resolve(slotSearchPath);
@ -330,7 +435,7 @@ public class NBIO implements NBPathsAPI.Facets {
// for testing
public LinkedHashSet<String> expandNamesAndSuffixes() {
return expandNamesAndSuffixes(names, new HashSet(extensions));
return expandNamesAndSuffixes(prefixes, names, extensionSets);
}
@ -339,46 +444,92 @@ public class NBIO implements NBPathsAPI.Facets {
* If no name is given, then <pre>{@code .*}</pre> is used.
* If suffixes are given, then all returned results must include at least
* one of the suffixes. If the name includes one of the suffixes given,
* then additional names are expanded to match the additional suffixes.
* then additional suffixes are added to the list of searchable path names.
*
* @param _names base filenames or path fragment, possibly fully-qualified
* @param _suffixes zero or more suffixes, which, if given, imply that one of them must match
* @param _names
* base filenames or path fragment, possibly fully-qualified
* @param _suffixCosetLists
* zero or more suffixes, which, if given, imply that one of them must match
* @return Expanded names of valid filename fragments according to the above rules
*/
public LinkedHashSet<String> expandNamesAndSuffixes(
public static LinkedHashSet<String> expandNamesAndSuffixes(
Set<String> _prefixes,
List<String> _names,
Set<String> _suffixes) {
List<Set<String>> _suffixCosetLists) {
LinkedHashSet<String> searches = new LinkedHashSet<>();
if (_names.size() == 0 && prefixes.size() == 0) {
searches.add(".*");
} else if (_names.size() > 0 && _suffixes.size() == 0) {
searches.addAll(_names);
} else if (_names.size() == 0 && _suffixes.size() > 0) {
_suffixes.stream().map(s -> ".*" + s).forEach(searches::add);
if (_names.size() > 0) {
if (_suffixCosetLists.size() > 0) { // names & suffixes
searches.addAll(expandSynonymPaths(_names, _suffixCosetLists));
} else { // names only
searches.addAll(_names);
}
} else {
for (String name : _names) {
// if (!name.equals(".*")) {
// searches.add(name);
// }
String basename = name;
boolean suffixed = false;
for (String suffix : _suffixes) {
if (name.endsWith(suffix)) {
suffixed = true;
basename = name.substring(0, name.length() - suffix.length());
break;
if (_suffixCosetLists.size() > 0) { // suffixes only
for (Set<String> suffixCoset : _suffixCosetLists) {
for (String suffix : suffixCoset) {
searches.add(".*" + suffix);
}
}
for (String suffix : _suffixes) {
searches.add(basename + suffix);
}
} else { // neither
searches.add(".*");
}
}
if (_prefixes.size() > 0) {
LinkedHashSet<String> prefixed = new LinkedHashSet<>(searches);
for (String prefix : _prefixes) {
for (String search : searches) {
prefixed.add(prefix + File.separator + search);
}
}
searches = prefixed;
}
return searches;
}
public static Set<String> expandSynonymPaths(List<String> names, List<Set<String>> suffixSets) {
if (suffixSets.size()==0) {
return new LinkedHashSet<>(names);
}
Set<String> newnames = new LinkedHashSet<>();
for (String name : names) {
Set<String> matchingSet = null;
String basename = null;
for (Set<String> suffixSet : suffixSets) {
for (String suffix : suffixSet) {
if (name.endsWith(suffix)) {
matchingSet = suffixSet;
basename = name.substring(0, name.length() - suffix.length());
// newnames.add(name); // Leave this here, it initializes precedence
break;
}
}
if (basename != null) {
break;
}
}
if (basename == null) {
if (name.contains(".") && !name.equals(".*")) {
// newnames.add(name);
suffixSets.stream().flatMap(s -> s.stream()).map(s -> name + s).forEach(newnames::add);
} else {
suffixSets.stream().flatMap(s -> s.stream()).map(s -> name + s).forEach(newnames::add);
}
} else {
for (String extension : matchingSet) {
newnames.add(basename + extension);
}
}
}
return newnames;
}
/**
* {@inheritDoc}
*/
@Override
public List<Content<?>> list() {
LinkedHashSet<String> searches = expandNamesAndSuffixes();
@ -413,6 +564,9 @@ public class NBIO implements NBPathsAPI.Facets {
return new ArrayList<>(foundFiles);
}
/**
* {@inheritDoc}
*/
@Override
public List<Path> relativeTo(String... base) {
String base1 = base[0];
@ -462,13 +616,16 @@ public class NBIO implements NBPathsAPI.Facets {
// }
/**
* {@inheritDoc}
*/
@Override
public String toString() {
return "NBIO{" +
"resolver=" + resolver +
", prefixes=" + prefixes +
", names=" + names +
", extensions=" + extensions +
", extensionSets=" + extensionSets +
'}';
}
}

55
nb-api/src/main/java/io/nosqlbench/api/content/NBIOSets.java Normal file
View File

@ -0,0 +1,55 @@
/*
* Copyright (c) 2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.api.content;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Set;
public class NBIOSets {
/**
* Combine overlapping sets or create new ones with no overlap
* @param setsData Existing sets
* @param newSets Additional sets
* @return combined sets
* @param <T>
*/
public static <T extends Comparable<T>> List<Set<T>> combine(List<Set<T>> setsData, Set<T>... newSets) {
for (Set<T> coset : newSets) {
Set<T> addTo = null;
for (Set<T> extensionSet : setsData) {
Set<T> union = new LinkedHashSet<>(coset);
for (T entry : coset) {
if (extensionSet.contains(entry)) {
addTo = extensionSet;
break;
}
}
if (addTo != null) {
break;
}
}
if (addTo==null) {
addTo=new LinkedHashSet<>();
setsData.add(addTo);
}
addTo.addAll(coset);
}
return setsData;
}
}

38
nb-api/src/main/java/io/nosqlbench/api/content/NBPathsAPI.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -19,11 +19,21 @@ package io.nosqlbench.api.content;
import java.nio.file.Path;
import java.util.List;
import java.util.Optional;
import java.util.Set;
/**
* This API provides a single service to locate and load content resource from multiple places,
* at build time and runtime, using a standard fluent-like pattern. The most important details for new uesrs are:
* <UL>
* <LI>Search paths are optional, and are tried <strong>after</strong> non-prefixed paths.</LI>
* <LI>If extensions are provided, then any result returned <strong>must</strong> match one of the extensions.</LI>
* <LI>The most efficient way to access a single path will be to use the {@link Facets#one()} method as {@link NBIO#one()}</LI>
* </UL>
*/
public interface NBPathsAPI {
interface Facets extends
GetSource, GetPrefix, GetName, GetExtension, DoSearch {
GetSource, GetPrefixes, GetPathname, GetExtensions, DoSearch {
}
interface GetSource {
@ -32,7 +42,7 @@ public interface NBPathsAPI {
*
* @return this builder
*/
GetPrefix localContent();
GetPrefixes localContent();
/**
* Only return content from remote URLs. If the user is providing non-URL content
@ -40,7 +50,7 @@ public interface NBPathsAPI {
*
* @return this builder
*/
GetPrefix remoteContent();
GetPrefixes remoteContent();
/**
* Only return content from the runtime classpath, internal resources that are bundled,
@ -48,14 +58,14 @@ public interface NBPathsAPI {
*
* @return this builder
*/
GetPrefix internalContent();
GetPrefixes internalContent();
/**
* Only return content from the filesystem, but not remote URLs nor internal bundled resources.
*
* @return this builder
*/
GetPrefix fileContent();
GetPrefixes fileContent();
/**
* Return content from everywhere, from remote URls, or from the file system and then the internal
@ -63,10 +73,10 @@ public interface NBPathsAPI {
*
* @return this builder
*/
GetPrefix allContent();
GetPrefixes allContent();
}
interface GetPrefix extends GetName {
interface GetPrefixes extends GetPathname {
/**
* Each of the prefix paths will be searched if the resource is not found with the exact
* path given. To be specific, if you want to search within a location based on wildcards,
@ -75,10 +85,10 @@ public interface NBPathsAPI {
* @param prefixPaths A list of paths to include in the search
* @return this builder
*/
GetPrefix prefix(String... prefixPaths);
GetPrefixes searchPrefixes(String... prefixPaths);
}
interface GetName extends GetExtension {
interface GetPathname extends GetExtensions {
/**
* Provide the names of the resources to be resolved. More than one resource may be provided.
* If no name is provided, then a wildcard search is assumed.
@ -86,7 +96,7 @@ public interface NBPathsAPI {
* @param name The name of the resource to load
* @return this builder
*/
GetExtension name(String... name);
GetExtensions pathname(String... name);
/**
* Provide a combined prefix, name and suffix in a combined form. For each search template provided,
@ -123,7 +133,7 @@ public interface NBPathsAPI {
}
interface GetExtension extends DoSearch {
interface GetExtensions extends DoSearch {
/**
* provide a list of optional file extensions which should be considered. If the content is
* not found under the provided name, then each of the extension is tried in order.
@ -134,8 +144,8 @@ public interface NBPathsAPI {
* @param extensions The extension names to try
* @return this builder
*/
DoSearch extension(String... extensions);
GetExtensions extensionSet(String... extensions);
GetExtensions extensionSets(Set<String>... cosets);
}
interface DoSearch {

29
nb-api/src/main/java/io/nosqlbench/api/content/URIResolver.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -22,6 +22,7 @@ import java.net.URI;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
/**
* This is a stateful search object for resources like Paths or URLs.
@ -48,6 +49,7 @@ public class URIResolver implements ContentResolver {
* Include resources from all known places, including remote URLs,
* the local default filesystem, and the classpath, which includes
* the jars that hold the current runtime application.
*
* @return this URISearch
*/
public URIResolver all() {
@ -57,6 +59,7 @@ public class URIResolver implements ContentResolver {
/**
* Includ resources in the default filesystem
*
* @return this URISearch
*/
public URIResolver inFS() {
@ -66,6 +69,7 @@ public class URIResolver implements ContentResolver {
/**
* Include resources in remote URLs
*
* @return this URISearch
*/
public URIResolver inURLs() {
@ -75,6 +79,7 @@ public class URIResolver implements ContentResolver {
/**
* Include resources within the classpath.
*
* @return this URISearch
*/
public URIResolver inCP() {
@ -117,26 +122,38 @@ public class URIResolver implements ContentResolver {
}
public URIResolver extension(String extension) {
this.extensions = this.extensions==null ? new ArrayList<>() : this.extensions;
this.extensions = this.extensions == null ? new ArrayList<>() : this.extensions;
this.extensions.add(extension);
return this;
}
public URIResolver extraPaths(String extraPath) {
this.extraPaths = this.extraPaths==null ? new ArrayList<>() : this.extraPaths;
this.extraPaths = this.extraPaths == null ? new ArrayList<>() : this.extraPaths;
this.extraPaths.add(Path.of(extraPath));
return this;
}
public Optional<Content<?>> resolveOneOptionally(String candidatePath) {
List<Content<?>> contents = resolveAll(candidatePath);
if (contents.size() == 1) {
return Optional.of(contents.get(0));
}
if (contents.size() == 0) {
return Optional.empty();
}
throw new BasicError("Error while loading content '" + candidatePath + "', only one is allowed, but " + contents.size() + " were found");
}
public Content<?> resolveOne(String candidatePath) {
List<Content<?>> contents = resolveAll(candidatePath);
if (contents.size()==1) {
if (contents.size() == 1) {
return contents.get(0);
}
if (contents.size()==0) {
if (contents.size() == 0) {
return null;
}
throw new BasicError("Error while loading content '" + candidatePath +"', only one is allowed, but " + contents.size() + " were found");
throw new BasicError("Error while loading content '" + candidatePath + "', only one is allowed, but " + contents.size() + " were found");
}
public String toString() {

4
nb-api/src/main/java/io/nosqlbench/api/markdown/providers/DocsRootDirectory.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -29,7 +29,7 @@ public abstract class DocsRootDirectory implements RawMarkdownSource {
@Override
public List<Content<?>> getMarkdownInfo() {
List<Content<?>> list = NBIO.local().name(getRootPathName()).list();
List<Content<?>> list = NBIO.local().pathname(getRootPathName()).list();
NBIOWalker.CollectVisitor capture = new NBIOWalker.CollectVisitor(true, false);
NBIOWalker.RegexFilter filter = new NBIOWalker.RegexFilter("\\.md",true);
for (Content<?> content : list) {

40
nb-api/src/test/java/io/nosqlbench/api/content/NBIOSetsTest.java Normal file
View File

@ -0,0 +1,40 @@
/*
* Copyright (c) 2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.api.content;
import org.junit.jupiter.api.Test;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;
import static org.assertj.core.api.Assertions.assertThat;
public class NBIOSetsTest {
@Test
public void testSetsAddition() {
List<Set<String>> data = new ArrayList<>();
data=NBIOSets.combine(data, Set.of("a","b"));
assertThat(data).isEqualTo(List.of(Set.of("a","b")));
data=NBIOSets.combine(data,Set.of("a","c"));
assertThat(data).isEqualTo(List.of(Set.of("a","b","c")));
data=NBIOSets.combine(data, Set.of("d"));
assertThat(data).isEqualTo(List.of(Set.of("a","b","c"),Set.of("d")));
}
}

161
nb-api/src/test/java/io/nosqlbench/nb/api/content/NBIOTest.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -26,13 +26,13 @@ import org.junit.jupiter.api.Test;
import java.io.File;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
import static org.assertj.core.api.Assertions.assertThat;
@ -42,72 +42,80 @@ public class NBIOTest {
@Test
public void testFullyQualifiedNameSearches() {
NBIO extensions = (NBIO) NBIO.all().name("foo.bar");
NBIO extensions = (NBIO) NBIO.all().pathname("foo.bar");
LinkedHashSet<String> searches = extensions.expandNamesAndSuffixes();
assertThat(searches).containsExactly("foo.bar");
}
@Test
public void testExpandWildcardAndExtensionsOnly() {
NBIO extensions = (NBIO) NBIO.all().name(".*").extension("foo","bar");
NBIO extensions = (NBIO) NBIO.all().pathname(".*").extensionSet("foo","bar");
LinkedHashSet<String> searches = extensions.expandNamesAndSuffixes();
assertThat(searches).contains(".*.foo",".*.bar");
}
@Test
public void testExpandNameAndAllSuffixesOnly() {
NBIO extensions = (NBIO) NBIO.all().name("foo.bar").extension("test1","test2");
NBIO extensions = (NBIO) NBIO.all().pathname("foo.bar").extensionSet("test1","test2");
LinkedHashSet<String> searches = extensions.expandNamesAndSuffixes();
assertThat(searches).containsExactly("foo.bar.test1","foo.bar.test2");
}
@Test
public void testExpandNamesAndExtensionsIfNotExtended() {
NBIO extensions = (NBIO) NBIO.all().name("foo").extension("baz","beez");
NBIO extensions = (NBIO) NBIO.all().pathname("foo").extensionSet("baz","beez");
LinkedHashSet<String> searches = extensions.expandNamesAndSuffixes();
assertThat(searches).contains("foo.baz","foo.beez");
}
@Test
public void testExpandNamesAndExtensionsAvoidsExtendedAlreadyExtended() {
NBIO extensions = (NBIO) NBIO.all().name("foo.baz").extension("baz","beez");
NBIO extensions = (NBIO) NBIO.all().pathname("foo.baz").extensionSet("baz","beez");
LinkedHashSet<String> searches = extensions.expandNamesAndSuffixes();
assertThat(searches).contains("foo.baz","foo.beez");
}
@Test
public void testExpandPrefixesAndFullName() {
NBIO extensions = (NBIO) NBIO.all().prefix("act1","act2").name("foo.bar");
NBIO extensions = (NBIO) NBIO.all().searchPrefixes("act1","act2").pathname("foo.bar");
LinkedHashSet<String> searches = extensions.expandNamesAndSuffixes();
assertThat(searches).containsExactly("foo.bar");
assertThat(searches).containsExactly("foo.bar","act1/foo.bar","act2/foo.bar");
}
@Test
public void testExpandAddExtensionNotNeeded() {
NBIO extensions = (NBIO) NBIO.all().name("foo.bar").extension("bar");
NBIO extensions = (NBIO) NBIO.all().pathname("foo.bar").extensionSet("bar");
LinkedHashSet<String> searches = extensions.expandNamesAndSuffixes();
assertThat(searches).containsExactly("foo.bar");
NBIO extensionsDot = (NBIO) NBIO.all().name("foo.bar").extension(".bar");
NBIO extensionsDot = (NBIO) NBIO.all().pathname("foo.bar").extensionSet(".bar");
LinkedHashSet<String> searchesDot = extensionsDot.expandNamesAndSuffixes();
assertThat(searchesDot).containsExactly("foo.bar");
}
@Test
public void testExpandExtensionCosets() {
Set<String> paths = NBIO.expandSynonymPaths(List.of("abc123.tubas"), List.of(Set.of(".foo")));
assertThat(paths).isEqualTo(Set.of("abc123.tubas.foo"));
paths = NBIO.expandSynonymPaths(List.of("abc123.tubas","def.123"), List.of(Set.of(".456",".789",".123")));
assertThat(paths).isEqualTo(Set.of("abc123.tubas.123","abc123.tubas.456","abc123.tubas.789","def.123","def.456","def.789"));
}
@Test
public void testExpandAddExtensionNeeded() {
NBIO extensions = (NBIO) NBIO.all().name("foo").extension("bar");
NBIO extensions = (NBIO) NBIO.all().pathname("foo").extensionSet("bar");
LinkedHashSet<String> searches = extensions.expandNamesAndSuffixes();
assertThat(searches).containsExactly("foo.bar");
NBIO extensionsDot = (NBIO) NBIO.all().name("foo").extension(".bar");
NBIO extensionsDot = (NBIO) NBIO.all().pathname("foo").extensionSet(".bar");
LinkedHashSet<String> searchesDot = extensionsDot.expandNamesAndSuffixes();
assertThat(searchesDot).containsExactly("foo.bar");
}
@Test
public void testLoadCsv1Classpath() {
NBPathsAPI.GetPrefix forSourceType = NBIO.classpath();
NBPathsAPI.GetName nesteddir1 = forSourceType.prefix("nesteddir1");
NBPathsAPI.GetExtension getExtension = nesteddir1.name("nesteddir2/testcsv1");
NBPathsAPI.DoSearch forCsvExtension = getExtension.extension(".csv");
NBPathsAPI.GetPrefixes forSourceType = NBIO.classpath();
NBPathsAPI.GetPathname nesteddir1 = forSourceType.searchPrefixes("nesteddir1");
NBPathsAPI.GetExtensions getExtensions = nesteddir1.pathname("nesteddir2/testcsv1");
NBPathsAPI.DoSearch forCsvExtension = getExtensions.extensionSet(".csv");
Optional<Content<?>> testcsv1 = forCsvExtension.first();
assertThat(testcsv1).isNotPresent();
@ -115,10 +123,10 @@ public class NBIOTest {
@Test
public void testLoadCsv1Filesystem() {
NBPathsAPI.GetPrefix forSourceType = NBIO.fs();
NBPathsAPI.GetName nesteddir1 = forSourceType.prefix("target/test-classes/nesteddir1");
NBPathsAPI.GetExtension getExtension = nesteddir1.name("nesteddir2/testcsv1");
NBPathsAPI.DoSearch forCsvExtension = getExtension.extension(".csv");
NBPathsAPI.GetPrefixes forSourceType = NBIO.fs();
NBPathsAPI.GetPathname nesteddir1 = forSourceType.searchPrefixes("target/test-classes/nesteddir1");
NBPathsAPI.GetExtensions getExtensions = nesteddir1.pathname("nesteddir2/testcsv1");
NBPathsAPI.DoSearch forCsvExtension = getExtensions.extensionSet(".csv");
Optional<Content<?>> testcsv1 = forCsvExtension.first();
assertThat(testcsv1).isNotPresent();
@ -130,7 +138,7 @@ public class NBIOTest {
@Test
public void testClasspathTestResource() {
List<List<Content<?>>> optionals =
NBIO.classpath().name("nesteddir1/nesteddir2/testcsv12.csv").resolveEach();
NBIO.classpath().pathname("nesteddir1/nesteddir2/testcsv12.csv").resolveEach();
assertThat(optionals).hasSize(1);
Content<?> content = optionals.get(0).get(0);
assertThat(content).isNotNull();
@ -139,9 +147,9 @@ public class NBIOTest {
@Test
public void testPathSearchForExtension() {
List<Content<?>> list = NBIO.classpath()
.prefix("nesteddir1")
.name(".*.csv")
.extension("csv")
.searchPrefixes("nesteddir1")
.pathname(".*.csv")
.extensionSet("csv")
.list();
assertThat(list).hasSize(2);
}
@ -149,9 +157,9 @@ public class NBIOTest {
@Test
public void testPathSearchForExtensionMissing() {
List<Content<?>> list = NBIO.classpath()
.prefix("nesteddir1")
.name(".*")
.extension("csv")
.searchPrefixes("nesteddir1")
.pathname(".*")
.extensionSet("csv")
.list();
assertThat(list).hasSize(2);
}
@ -159,9 +167,9 @@ public class NBIOTest {
@Test
public void testPathSearchForMultipleExtensions() {
List<Content<?>> list = NBIO.classpath()
.prefix("nesteddir1")
.name(".*")
.extension("csv","txt")
.searchPrefixes("nesteddir1")
.pathname(".*")
.extensionSet("csv","txt")
.list();
assertThat(list).hasSize(3);
}
@ -169,9 +177,9 @@ public class NBIOTest {
@Test
public void testPathSearchForSuffix() {
List<Content<?>> list = NBIO.classpath()
.prefix("nesteddir1")
.name("nesteddir2/testdata12")
.extension("txt")
.searchPrefixes("nesteddir1")
.pathname("nesteddir2/testdata12")
.extensionSet("txt")
.list();
assertThat(list).hasSize(1);
}
@ -179,8 +187,8 @@ public class NBIOTest {
@Test
public void testPathSearchInDifferentVantagePoints() {
List<Path> list = NBIO.fs()
.prefix("target/test-classes/nesteddir1")
.extension("csv")
.searchPrefixes("target/test-classes/nesteddir1")
.extensionSet("csv")
.list().stream().map(Content::asPath)
.collect(Collectors.toList());
@ -193,19 +201,19 @@ public class NBIOTest {
@Test
public void testLoadNamedFileAsYmlExtension() {
List<Content<?>> list = NBIO.classpath()
.name("nesteddir1/nesteddir2/testworkload1")
.extension("yml")
.pathname("nesteddir1/nesteddir2/testworkload1")
.extensionSet("yml")
.list();
assertThat(list).hasSize(1);
list = NBIO.classpath()
.name("nesteddir1/nesteddir2/testworkload1.yml")
.pathname("nesteddir1/nesteddir2/testworkload1.yml")
.list();
assertThat(list).hasSize(1);
list = NBIO.classpath()
.name("nesteddir1/nesteddir2/testworkload1")
.extension("abc","yml")
.pathname("nesteddir1/nesteddir2/testworkload1")
.extensionSet("abc","yml")
.list();
assertThat(list).hasSize(1);
}
@ -214,15 +222,15 @@ public class NBIOTest {
public void testLoadAllFilesUnderPath() {
List<Content<?>> list = null;
list = NBIO.classpath().prefix("./").list();
list = NBIO.classpath().searchPrefixes("./").list();
logger.debug("found " + list.size() + " entries for path '.'");
assertThat(list).hasSizeGreaterThan(0);
list = NBIO.fs().prefix("./").list();
list = NBIO.fs().searchPrefixes("./").list();
logger.debug("found " + list.size() + " entries for path '.'");
assertThat(list).hasSizeGreaterThan(0);
list = NBIO.remote().prefix("./").list();
list = NBIO.remote().searchPrefixes("./").list();
logger.debug("found " + list.size() + " entries for path '.'");
assertThat(list).hasSize(0);
}
@ -230,17 +238,17 @@ public class NBIOTest {
@Test
public void test() {
List<Content<?>> list = NBIO.fs()
.prefix(Paths.get("target/test-classes/").toString())
.name("gamma.yaml").list();
.searchPrefixes(Paths.get("target/test-classes/").toString())
.pathname("gamma.yaml").list();
assertThat(list).hasSize(1);
}
@Test
public void testWildcardFilenameMatch() {
NBPathsAPI.DoSearch gammasSearch = NBIO.all()
.prefix(Paths.get("target/test-classes/").toString())
.name(".*gamma")
.extension("yaml");
.searchPrefixes(Paths.get("target/test-classes/").toString())
.pathname(".*gamma")
.extensionSet("yaml");
List<Content<?>> gammas = gammasSearch.list();
assertThat(gammas).hasSize(3);
}
@ -249,33 +257,57 @@ public class NBIOTest {
@Test
public void testSpecificFilenameMatch() {
NBPathsAPI.DoSearch gammasSearch = NBIO.all()
.prefix(Paths.get("target/test-classes/").toString())
.name("gamma")
.extension("yaml");
.searchPrefixes(Paths.get("target/test-classes/").toString())
.pathname("gamma")
.extensionSet("yaml");
List<Content<?>> gammas = gammasSearch.list();
assertThat(gammas).hasSize(1);
}
@Test
public void onlyMatchExtensionFilesWhenExtensionsProvided() {
public void matchOneWithoutTryingPrefixesFirst() {
Content<?> result = NBIO.all()
.searchPrefixes(
Paths.get("target/test-classes/nesteddir1/nesteddir2").toString()
)
.pathname("nesteddir1/alpha-gamma.yaml")
.one();
assertThat(result).isNotNull();
assertThat(result.getURI().toString()).matches(".*?[^1]/nesteddir1/alpha-gamma.yaml");
}
@Test
public void matchOneFallsThroughToPrefixesSecond() {
Content<?> result = NBIO.all()
.searchPrefixes(
Paths.get("target/test-classes/nesteddir1/nesteddir2").toString()
)
.pathname("alpha-gamma.yaml")
.one();
assertThat(result).isNotNull();
assertThat(result.getURI().toString()).matches(".*?nesteddir1/nesteddir2/alpha-gamma.yaml");
}
@Test
public void onlyMatchExtensionFilesWhenExtensionInCoset() {
// This search is invalid because by providing extensions, all results
// are required to match one of the extensions, thus the only valid
// match here would be alpha-gamma.yaml.js
NBPathsAPI.DoSearch invalidSearch = NBIO.all()
.prefix(Paths.get("target/test-classes/").toString())
.name("alpha-gamma.yaml")
.extension("js");
.searchPrefixes(Paths.get("target/test-classes/").toString())
.pathname("alpha-gamma.yaml")
.extensionSet("js");
NBPathsAPI.DoSearch validSearch1 = NBIO.all()
.prefix(Paths.get("target/test-classes/").toString())
.name("alpha-gamma")
.extension("js");
.searchPrefixes(Paths.get("target/test-classes/").toString())
.pathname("alpha-gamma")
.extensionSet("js");
NBPathsAPI.DoSearch validSearch2 = NBIO.all()
.prefix(Paths.get("target/test-classes/").toString())
.name("alpha-gamma.js")
.extension();
.searchPrefixes(Paths.get("target/test-classes/").toString())
.pathname("alpha-gamma.js")
.extensionSet();
assertThat(invalidSearch.list()).hasSize(0);
@ -292,12 +324,13 @@ public class NBIOTest {
File tempFile = File.createTempFile(tmpdir.toString(), "testfile.csv");
tempFile.deleteOnExit();
String fullpath = tempFile.getAbsolutePath();
Files.write(Path.of(fullpath), "COL1,COL2\n\"val1\",\"val2\"\n".getBytes(StandardCharsets.UTF_8));
List<Content<?>> results = NBIO.all().name(fullpath).list();
Files.writeString(Path.of(fullpath), "COL1,COL2\n\"val1\",\"val2\"\n");
List<Content<?>> results = NBIO.all().pathname(fullpath).list();
assertThat(results.size()).isEqualTo(1);
} catch (IOException e) {
throw new RuntimeException(e);
}
}
}

2
nbr/src/main/java/io/nosqlbench/api/docsapi/docexporter/BundledVersionManifest.java
View File

@ -28,7 +28,7 @@ public class BundledVersionManifest implements BundledMarkdownManifest {
public DocsBinder getDocs() {
return new Docs()
.namespace("versions")
.addPath(NBIO.classpath().name("nb_version_info.md").one().asPath())
.addPath(NBIO.classpath().pathname("nb_version_info.md").one().asPath())
.asDocsBinder();
}
}