IntenseFinance/nosqlbench
3
0
Fork 0
mirror of https://github.com/nosqlbench/nosqlbench.git synced 2025-02-25 18:55:28 -06:00
Code Issues Packages Projects Releases Wiki Activity

first part of merge

Browse Source
This commit is contained in:
MikeYaacoubStax 2023-02-21 15:36:24 -05:00
commit 90815e237d
50 changed files with 657 additions and 406 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
adapter-cqld4/pom.xml
View File

@ -55,7 +55,7 @@
<dependency>
<groupId>org.apache.tinkerpop</groupId>
<artifactId>tinkergraph-gremlin</artifactId>
<version>3.6.1</version>
<version>3.6.2</version>
</dependency>
<dependency>
@ -79,7 +79,7 @@
<dependency>
<groupId>org.xerial.snappy</groupId>
<artifactId>snappy-java</artifactId>
<version>1.1.8.4</version>
<version>1.1.9.0</version>
</dependency>
<dependency>

8
adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/Cqld4Space.java
View File

@ -99,7 +99,7 @@ public class Cqld4Space implements AutoCloseable {
int port = cfg.getOptional(int.class, "port").orElse(9042);
Optional<String> scb = cfg.getOptional(String.class, "secureconnectbundle", "scb");
scb.flatMap(s -> NBIO.all().name(s).first().map(Content::getInputStream))
scb.flatMap(s -> NBIO.all().pathname(s).first().map(Content::getInputStream))
.map(builder::withCloudSecureConnectBundle);
Optional<List<InetSocketAddress>> contactPointsOption = cfg
@ -229,14 +229,14 @@ public class Cqld4Space implements AutoCloseable {
for (String loaderspec : loaderspecs) {
// path
Optional<Content<?>> fsconfig = NBIO.fs().name(driverconfig).first();
Optional<Content<?>> fsconfig = NBIO.fs().pathname(driverconfig).first();
if (fsconfig.isPresent()) {
loaders.add(DriverConfigLoader.fromPath(fsconfig.get().asPath()));
continue;
}
// classpath
Optional<Content<?>> cpconfig = NBIO.classpath().name(driverconfig).first();
Optional<Content<?>> cpconfig = NBIO.classpath().pathname(driverconfig).first();
if (cpconfig.isPresent()) {
loaders.add(DriverConfigLoader.fromClasspath(driverconfig));
continue;
@ -244,7 +244,7 @@ public class Cqld4Space implements AutoCloseable {
// URLs
try {
Optional<Content<?>> removeconf = NBIO.remote().name(driverconfig).first();
Optional<Content<?>> removeconf = NBIO.remote().pathname(driverconfig).first();
if (removeconf.isPresent()) {
loaders.add(DriverConfigLoader.fromUrl(removeconf.get().getURI().toURL()));
continue;

10
adapter-cqld4/src/main/java/io/nosqlbench/adapter/cqld4/opdispensers/Cqld4BaseOpDispenser.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -86,10 +86,10 @@ public abstract class Cqld4BaseOpDispenser extends BaseOpDispenser<Cqld4CqlOp, C
protected LongFunction<Statement> getEnhancedStmtFunc(LongFunction<Statement> basefunc, ParsedOp op) {
LongFunction<Statement> partial = basefunc;
partial = op.enhanceEnum(partial, "cl", DefaultConsistencyLevel.class, Statement::setConsistencyLevel);
partial = op.enhanceEnum(partial, "consistency_level", DefaultConsistencyLevel.class, Statement::setConsistencyLevel);
partial = op.enhanceEnum(partial, "scl", DefaultConsistencyLevel.class, Statement::setSerialConsistencyLevel);
partial = op.enhanceEnum(partial, "serial_consistency_level", DefaultConsistencyLevel.class, Statement::setSerialConsistencyLevel);
partial = op.enhanceEnumOptionally(partial, "cl", DefaultConsistencyLevel.class, Statement::setConsistencyLevel);
partial = op.enhanceEnumOptionally(partial, "consistency_level", DefaultConsistencyLevel.class, Statement::setConsistencyLevel);
partial = op.enhanceEnumOptionally(partial, "scl", DefaultConsistencyLevel.class, Statement::setSerialConsistencyLevel);
partial = op.enhanceEnumOptionally(partial, "serial_consistency_level", DefaultConsistencyLevel.class, Statement::setSerialConsistencyLevel);
partial = op.enhanceFuncOptionally(partial, "idempotent", Boolean.class, Statement::setIdempotent);
partial = op.enhanceFuncOptionally(partial, "timeout", double.class, (statement, l) -> statement.setTimeout(Duration.ofMillis((long) (l * 1000L))));
partial = op.enhanceFuncOptionally(partial, "custom_payload", Map.class, Statement::setCustomPayload);

4
adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/core/CGDefaultCqlBindings.java
View File

@ -37,9 +37,9 @@ public class CGDefaultCqlBindings implements BindingsLibrary {
public CGDefaultCqlBindings() {
Content<?> content = NBIO.all()
.name(DEFAULT_BINDINGS_FILE)
.pathname(DEFAULT_BINDINGS_FILE)
.first()
.or(() -> NBIO.all().prefix(DEFAULT_CFG_DIR).name(DEFAULT_BINDINGS_FILE).first())
.or(() -> NBIO.all().searchPrefixes(DEFAULT_CFG_DIR).pathname(DEFAULT_BINDINGS_FILE).first())
.orElseThrow(
() -> new RuntimeException("Unable to load " + DEFAULT_BINDINGS_FILE +
", from local dir or internally as cqlgen" + DEFAULT_BINDINGS_FILE)

4
adapter-cqld4/src/main/java/io/nosqlbench/cqlgen/core/CGWorkloadExporter.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -125,7 +125,7 @@ public class CGWorkloadExporter implements BundledApp {
Yaml yaml = new Yaml();
CGWorkloadExporter exporter;
Content<?> cqlgencfg = NBIO.local().prefix("cqlgen").name("cqlgen").extension("conf").first().orElseThrow();
Content<?> cqlgencfg = NBIO.local().searchPrefixes("cqlgen").pathname("cqlgen").extensionSet("conf").first().orElseThrow();
if (cqlgencfg == null) {
throw new RuntimeException("Unable to load cqlgen.conf");
}

2
adapter-cqld4/src/main/resources/activities/baselines/cql-iot-dse.yaml
View File

@ -52,7 +52,7 @@ blocks:
prepared: false
ops:
truncate-table: |
truncate table <<keyspace:baselines>>.<<table:iot>>;
truncate table TEMPLATE(keyspace, baselines).TEMPLATE(table:iot);
rampup:
params:
cl: <<write_cl:LOCAL_QUORUM>>

2
adapter-dynamodb/pom.xml
View File

@ -43,7 +43,7 @@
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk-dynamodb</artifactId>
<version>1.12.386</version>
<version>1.12.395</version>
</dependency>
</dependencies>

4
adapter-http/src/main/java/io/nosqlbench/adapter/http/statuscodes/HttpStatusCodes.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -30,7 +30,7 @@ public class HttpStatusCodes {
private static final IetfStatusCode[] codes = loadMap();
private static IetfStatusCode[] loadMap() {
Content<?> csv = NBIO.local().name("ietf-http-status-codes").extension("csv").one();
Content<?> csv = NBIO.local().pathname("ietf-http-status-codes").extensionSet("csv").one();
InputStreamReader isr = new InputStreamReader(csv.getInputStream());
IetfStatusCode[] codes = new IetfStatusCode[600];

2
adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/OpsLoader.java
View File

@ -54,7 +54,7 @@ public class OpsLoader {
public static OpsDocList loadPath(String path, Map<String, ?> params, String... searchPaths) {
String[] extensions = path.indexOf('.')>-1 ? new String[]{} : YAML_EXTENSIONS;
Content<?> foundPath = NBIO.all().prefix(searchPaths).name(path).extension(extensions).first()
Content<?> foundPath = NBIO.all().searchPrefixes(searchPaths).pathname(path).extensionSet(extensions).first()
.orElseThrow(() -> new RuntimeException("Unable to load path '" + path + "'"));
OpTemplateFormat fmt = OpTemplateFormat.valueOfURI(foundPath.getURI());
return loadString(foundPath.asString(), fmt, params, foundPath.getURI());

2
adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawOpsLoader.java
View File

@ -69,7 +69,7 @@ public class RawOpsLoader {
String data = null;
try {
Optional<Content<?>> oyaml = NBIO.all().prefix(searchPaths).name(path).extension(YAML_EXTENSIONS).first();
Optional<Content<?>> oyaml = NBIO.all().searchPrefixes(searchPaths).pathname(path).extensionSet(YAML_EXTENSIONS).first();
data = oyaml.map(Content::asString).orElseThrow(() -> new BasicError("Unable to load " + path));
return loadString(data);
} catch (Exception e) {

2
adapters-api/src/main/java/io/nosqlbench/engine/api/activityconfig/rawyaml/RawYamlLoader.java
View File

@ -65,7 +65,7 @@ public class RawYamlLoader {
String data = null;
try {
Optional<Content<?>> oyaml = NBIO.all().prefix(searchPaths).name(path).extension(YAML_EXTENSIONS).first();
Optional<Content<?>> oyaml = NBIO.all().searchPrefixes(searchPaths).pathname(path).extensionSet(YAML_EXTENSIONS).first();
data = oyaml.map(Content::asString).orElseThrow(() -> new BasicError("Unable to load " + path));
return loadString(logger, data);
} catch (Exception e) {

1
adapters-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/BaseDriverAdapter.java
View File

@ -160,6 +160,7 @@ public abstract class BaseDriverAdapter<R extends Op, S> implements DriverAdapte
.add(Param.optional("stride").setRegex("\\d+"))
.add(Param.optional("striderate", String.class, "rate limit for strides per second"))
.add(Param.optional("cycles").setRegex("\\d+[KMBGTPE]?|\\d+[KMBGTPE]?\\.\\.\\d+[KMBGTPE]?").setDescription("cycle interval to use"))
.add(Param.optional("recycles").setDescription("allow cycles to be re-used this many times"))
.add(Param.optional(List.of("cyclerate", "targetrate", "rate"), String.class, "rate limit for cycles per second"))
.add(Param.optional("phaserate", String.class, "rate limit for phases per second"))
.add(Param.optional("seq", String.class, "sequencing algorithm"))

10
adapters-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/DriverAdapter.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -28,15 +28,11 @@ import io.nosqlbench.api.config.standard.NBConfiguration;
import io.nosqlbench.api.content.Content;
import io.nosqlbench.api.content.NBIO;
import java.io.File;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.function.Function;
import java.util.function.LongFunction;
import java.util.stream.Stream;
/**
* <P>The DriverAdapter interface is expected to be the replacement
@ -185,10 +181,10 @@ public interface DriverAdapter<OPTYPE extends Op, SPACETYPE> {
String dev_docspath = "adapter-" + this.getAdapterName() + "/src/main/resources/docs/" + this.getAdapterName();
String cp_docspath = "docs/" + this.getAdapterName();
Optional<Content<?>> bundled_docs = NBIO.local().name(dev_docspath, cp_docspath).first();
Optional<Content<?>> bundled_docs = NBIO.local().pathname(dev_docspath, cp_docspath).first();
bundled_docs.map(Content::asPath).ifPresent(docs::addContentsOf);
Optional<Content<?>> maindoc = NBIO.local().name("/src/main/resources/" + this.getAdapterName() + ".md", this.getAdapterName() + ".md").first();
Optional<Content<?>> maindoc = NBIO.local().pathname("/src/main/resources/" + this.getAdapterName() + ".md", this.getAdapterName() + ".md").first();
maindoc.map(Content::asPath).ifPresent(docs::addPath);

2
adapters-api/src/main/java/io/nosqlbench/engine/api/specifications/SpecDocsManifest.java
View File

@ -30,7 +30,7 @@ public class SpecDocsManifest implements BundledMarkdownManifest {
@Override
public DocsBinder getDocs() {
Docs docs = new Docs().namespace("workload_definition");
List<Content<?>> specfiles = NBIO.classpath().prefix("workload_definition/").extension(".md").list();
List<Content<?>> specfiles = NBIO.classpath().searchPrefixes("workload_definition/").extensionSet(".md").list();
for (Content<?> specfile : specfiles) {
docs.addPath(specfile.asPath());
}

4
adapters-api/src/main/java/io/nosqlbench/engine/api/templating/ParsedOp.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -893,7 +893,7 @@ public class ParsedOp implements LongFunction<Map<String, ?>>, StaticFieldReader
* @param <FE> The enhancer function result type
* @return an (optionally) enhanced base function
*/
public <FA,FE extends Enum<FE>> LongFunction<FA> enhanceEnum(
public <FA,FE extends Enum<FE>> LongFunction<FA> enhanceEnumOptionally(
LongFunction<FA> func,
String field,
Class<FE> type,

3
adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/OpsLoaderTest.java
View File

@ -20,6 +20,7 @@ import io.nosqlbench.engine.api.activityconfig.yaml.OpsDoc;
import io.nosqlbench.engine.api.activityconfig.yaml.OpsDocList;
import org.junit.jupiter.api.Test;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@ -29,7 +30,7 @@ public class OpsLoaderTest {
@Test
public void testTemplateVarSubstitution() {
OpsDocList opsDocs = OpsLoader.loadPath("activities/template_vars", Map.of(),"src/test/resources");
OpsDocList opsDocs = OpsLoader.loadPath("activities/template_vars", new HashMap<>(),"src/test/resources");
assertThat(opsDocs).isNotNull();
List<OpsDoc> docs = opsDocs.getStmtDocs();
assertThat(docs).hasSize(1);

3
adapters-api/src/test/java/io/nosqlbench/engine/api/activityconfig/specifications/YamlSpecValidator.java
View File

@ -33,6 +33,7 @@ import org.apache.logging.log4j.Logger;
import org.assertj.core.api.Assertions;
import java.lang.reflect.Type;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
@ -108,7 +109,7 @@ public class YamlSpecValidator implements STAssemblyValidator {
}.getType();
List<Map<String, Object>> expectedList = gson.fromJson(json, type);
OpsDocList stmtsDocs = OpsLoader.loadString(yaml, OpTemplateFormat.yaml, Map.of(), null);
OpsDocList stmtsDocs = OpsLoader.loadString(yaml, OpTemplateFormat.yaml, new HashMap<>(), null);
List<OpTemplate> stmts = stmtsDocs.getOps();
List<Map<String, Object>> stmt_objs = stmts.stream().map(OpTemplate::asData).collect(Collectors.toList());

6
docsys/pom.xml
View File

@ -100,18 +100,18 @@
<dependency>
<groupId>com.fasterxml.jackson.jaxrs</groupId>
<artifactId>jackson-jaxrs-json-provider</artifactId>
<version>2.14.1</version>
<version>2.14.2</version>
</dependency>
<dependency>
<groupId>com.sun.xml.bind</groupId>
<artifactId>jaxb-core</artifactId>
<version>4.0.1</version>
<version>4.0.2</version>
</dependency>
<dependency>
<groupId>com.sun.xml.bind</groupId>
<artifactId>jaxb-impl</artifactId>
<version>4.0.1</version>
<version>4.0.2</version>
</dependency>
<dependency>

8
engine-api/pom.xml
View File

@ -135,6 +135,14 @@
<directory>src/test/resources</directory>
<filtering>false</filtering> <!-- exclusion from defaults -->
</testResource>
<testResource>
<filtering>true</filtering>
<directory>src/test/resources</directory>
<includes>
<include>log4j2-test.xml</include>
</includes>
</testResource>
</testResources>
</build>

19
engine-api/src/main/java/io/nosqlbench/engine/api/activityimpl/uniform/StandardActivity.java
View File

@ -18,6 +18,7 @@ package io.nosqlbench.engine.api.activityimpl.uniform;
import io.nosqlbench.api.config.standard.*;
import io.nosqlbench.api.engine.activityimpl.ActivityDef;
import io.nosqlbench.api.errors.BasicError;
import io.nosqlbench.api.errors.OpConfigError;
import io.nosqlbench.engine.api.activityapi.planning.OpSequence;
import io.nosqlbench.engine.api.activityconfig.OpsLoader;
@ -66,9 +67,14 @@ public class StandardActivity<R extends Op, S> extends SimpleActivity implements
}
ServiceLoader<DriverAdapter> adapterLoader = ServiceLoader.load(DriverAdapter.class);
Optional<DriverAdapter> defaultAdapter = activityDef.getParams().getOptionalString("driver")
Optional<String> defaultDriverName = activityDef.getParams().getOptionalString("driver");
Optional<DriverAdapter> defaultAdapter = defaultDriverName
.flatMap(s -> ServiceSelector.of(s, adapterLoader).get());
if (defaultDriverName.isPresent() && defaultAdapter.isEmpty()) {
throw new BasicError("Unable to load default driver adapter '" + defaultDriverName.get() + "'");
}
// HERE, op templates are loaded before drivers are loaded
List<OpTemplate> opTemplates = loadOpTemplates(defaultAdapter);
@ -77,11 +83,12 @@ public class StandardActivity<R extends Op, S> extends SimpleActivity implements
List<DriverAdapter> adapterlist = new ArrayList<>();
NBConfigModel supersetConfig = ConfigModel.of(StandardActivity.class).add(yamlmodel);
Optional<String> defaultDriverOption = activityDef.getParams().getOptionalString("driver");
for (OpTemplate ot : opTemplates) {
ParsedOp incompleteOpDef = new ParsedOp(ot, NBConfiguration.empty(), List.of());
String driverName = incompleteOpDef.takeOptionalStaticValue("driver", String.class)
.or(() -> incompleteOpDef.takeOptionalStaticValue("type",String.class))
.or(() -> activityDef.getParams().getOptionalString("driver"))
.or(() -> defaultDriverOption)
.orElseThrow(() -> new OpConfigError("Unable to identify driver name for op template:\n" + ot));
// String driverName = ot.getOptionalStringParam("driver")
@ -109,7 +116,6 @@ public class StandardActivity<R extends Op, S> extends SimpleActivity implements
}
supersetConfig.assertValidConfig(activityDef.getParams().getStringStringMap());
DriverAdapter adapter = adapters.get(driverName);
adapterlist.add(adapter);
ParsedOp pop = new ParsedOp(ot, adapter.getConfiguration(), List.of(adapter.getPreprocessor()));
@ -117,6 +123,13 @@ public class StandardActivity<R extends Op, S> extends SimpleActivity implements
pops.add(pop);
}
if (defaultDriverOption.isPresent()) {
long matchingDefault = mappers.keySet().stream().filter(n -> n.equals(defaultDriverOption.get())).count();
if (matchingDefault==0) {
logger.warn("All op templates used a different driver than the default '" + defaultDriverOption.get()+"'");
}
}
try {
boolean strict = activityDef.getParams().getOptionalBoolean("strict").orElse(false);
sequence = createOpSourceFromParsedOps(adapters, mappers, adapterlist, pops);

48
engine-api/src/main/java/io/nosqlbench/engine/api/scenarios/NBCLIScenarioParser.java
View File

@ -50,10 +50,10 @@ public class NBCLIScenarioParser {
public static boolean isFoundWorkload(String workload, String... includes) {
Optional<Content<?>> found = NBIO.all()
.prefix("activities")
.prefix(includes)
.name(workload)
.extension(RawOpsLoader.YAML_EXTENSIONS)
.searchPrefixes("activities")
.searchPrefixes(includes)
.pathname(workload)
.extensionSet(RawOpsLoader.YAML_EXTENSIONS)
.first();
return found.isPresent();
}
@ -64,10 +64,10 @@ public class NBCLIScenarioParser {
String workloadName = arglist.removeFirst();
Optional<Content<?>> found = NBIO.all()
.prefix("activities")
.prefix(includes)
.name(workloadName)
.extension(RawOpsLoader.YAML_EXTENSIONS)
.searchPrefixes("activities")
.searchPrefixes(includes)
.pathname(workloadName)
.extensionSet(RawOpsLoader.YAML_EXTENSIONS)
.first();
//
Content<?> workloadContent = found.orElseThrow();
@ -108,10 +108,10 @@ public class NBCLIScenarioParser {
// Load in named scenario
Content<?> yamlWithNamedScenarios = NBIO.all()
.prefix(SEARCH_IN)
.prefix(includes)
.name(workloadName)
.extension(RawOpsLoader.YAML_EXTENSIONS)
.searchPrefixes(SEARCH_IN)
.searchPrefixes(includes)
.pathname(workloadName)
.extensionSet(RawOpsLoader.YAML_EXTENSIONS)
.first().orElseThrow();
// TODO: The yaml needs to be parsed with arguments from each command independently to support template vars
OpsDocList scenariosYaml = OpsLoader.loadContent(yamlWithNamedScenarios, new LinkedHashMap<>(userProvidedParams));
@ -228,7 +228,7 @@ public class NBCLIScenarioParser {
private static LinkedHashMap<String, CmdArg> parseStep(String cmd) {
LinkedHashMap<String, CmdArg> parsedStep = new LinkedHashMap<>();
String[] namedStepPieces = cmd.split(" ");
String[] namedStepPieces = cmd.split(" +");
for (String commandFragment : namedStepPieces) {
Matcher matcher = WordAndMaybeAssignment.matcher(commandFragment);
@ -320,8 +320,8 @@ public class NBCLIScenarioParser {
}
}
Content<?> content = NBIO.all().prefix(SEARCH_IN)
.name(referenced).extension(RawOpsLoader.YAML_EXTENSIONS)
Content<?> content = NBIO.all().searchPrefixes(SEARCH_IN)
.pathname(referenced).extensionSet(RawOpsLoader.YAML_EXTENSIONS)
.one();
OpsDocList stmts = null;
@ -379,14 +379,14 @@ public class NBCLIScenarioParser {
public static List<WorkloadDesc> getWorkloadsWithScenarioScripts(boolean defaultIncludes, String... includes) {
NBPathsAPI.GetPrefix searchin = NBIO.all();
NBPathsAPI.GetPrefixes searchin = NBIO.all();
if (defaultIncludes) {
searchin = searchin.prefix(SEARCH_IN);
searchin = searchin.searchPrefixes(SEARCH_IN);
}
List<Content<?>> activities = searchin
.prefix(includes)
.extension(RawOpsLoader.YAML_EXTENSIONS)
.searchPrefixes(includes)
.extensionSet(RawOpsLoader.YAML_EXTENSIONS)
.list();
return filterForScenarios(activities);
@ -395,15 +395,15 @@ public class NBCLIScenarioParser {
public static List<String> getScripts(boolean defaultIncludes, String... includes) {
NBPathsAPI.GetPrefix searchin = NBIO.all();
NBPathsAPI.GetPrefixes searchin = NBIO.all();
if (defaultIncludes) {
searchin = searchin.prefix(SEARCH_IN);
searchin = searchin.searchPrefixes(SEARCH_IN);
}
List<Path> scriptPaths = searchin
.prefix("scripts/auto")
.prefix(includes)
.extension("js")
.searchPrefixes("scripts/auto")
.searchPrefixes(includes)
.extensionSet("js")
.list().stream().map(Content::asPath).collect(Collectors.toList());
List<String> scriptNames = new ArrayList();

2
engine-cli/src/main/java/io/nosqlbench/engine/cli/BasicScriptBuffer.java
View File

@ -145,7 +145,7 @@ public class BasicScriptBuffer implements ScriptBuffer {
logger.debug(() -> "Looking for " + script_path);
Content<?> one = NBIO.all().prefix("scripts").name(script_path).extension("js").one();
Content<?> one = NBIO.all().searchPrefixes("scripts").pathname(script_path).extensionSet("js").one();
scriptData = one.asString();
StrInterpolator interpolator = new StrInterpolator(cmd.getParams());

12
engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLI.java
View File

@ -306,16 +306,16 @@ public class NBCLI implements Function<String[], Integer> {
logger.debug(() -> "user requests to copy out " + resourceToCopy);
Optional<Content<?>> tocopy = NBIO.classpath()
.prefix("activities")
.prefix(options.wantsIncludes())
.name(resourceToCopy).extension(RawOpsLoader.YAML_EXTENSIONS).first();
.searchPrefixes("activities")
.searchPrefixes(options.wantsIncludes())
.pathname(resourceToCopy).extensionSet(RawOpsLoader.YAML_EXTENSIONS).first();
if (tocopy.isEmpty()) {
tocopy = NBIO.classpath()
.prefix().prefix(options.wantsIncludes())
.prefix(options.wantsIncludes())
.name(resourceToCopy).first();
.searchPrefixes().searchPrefixes(options.wantsIncludes())
.searchPrefixes(options.wantsIncludes())
.pathname(resourceToCopy).first();
}
Content<?> data = tocopy.orElseThrow(

6
engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLICommandParser.java
View File

@ -71,9 +71,9 @@ public class NBCLICommandParser {
break;
default:
Optional<Content<?>> scriptfile = NBIO.local()
.prefix("scripts/auto")
.name(word)
.extension("js")
.searchPrefixes("scripts/auto")
.pathname(word)
.extensionSet("js")
.first();
//Script

10
engine-cli/src/main/java/io/nosqlbench/engine/cli/PathCanonicalizer.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -35,9 +35,9 @@ public class PathCanonicalizer {
public String canonicalizePath(String path) {
Optional<Content<?>> found = NBIO.local().prefix("activities")
.prefix(includes)
.name(path)
Optional<Content<?>> found = NBIO.local().searchPrefixes("activities")
.searchPrefixes(includes)
.pathname(path)
.first();
if (found.isPresent()) {
@ -46,7 +46,7 @@ public class PathCanonicalizer {
rewriteTo=(rewriteTo.startsWith(separator) ? rewriteTo.substring(1) : rewriteTo);
if (!rewriteTo.equals(path)) {
if (NBIO.local().prefix("activities").prefix(includes).name(rewriteTo).first().isPresent()) {
if (NBIO.local().searchPrefixes("activities").searchPrefixes(includes).pathname(rewriteTo).first().isPresent()) {
logger.info("rewrote path for " + path + " as " + rewriteTo);
return rewriteTo;
} else {

6
engine-cli/src/test/resources/activities/scenario-test.yaml
View File

@ -2,9 +2,9 @@ min_version: "5.17.1"
scenarios:
default:
schema: run driver==stdout workload===scenario-test tags=block:"schema.*"
rampup: run driver=stdout workload===scenario-test tags=block:rampup cycles=TEMPLATE(cycles1,10)
main: run driver=stdout workload===scenario-test tags=block:"main.*" cycles=TEMPLATE(cycles2,10)
schema: run driver==stdout workload===scenario-test tags=block:"schema.*"
rampup: run driver=stdout workload===scenario-test tags=block:rampup cycles=TEMPLATE(cycles1,10)
main: run driver=stdout workload===scenario-test tags=block:"main.*" cycles=TEMPLATE(cycles2,10)
schema-only:
schema: run driver=stdout workload==scenario-test tags=block:"schema.*" doundef==undef

4
engine-clients/src/test/java/io/nosqlbench/engine/clients/prometheus/PMatrixElemTest.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -33,7 +33,7 @@ public class PMatrixElemTest {
@Disabled
public void testMatrixElem() {
Gson gson = new GsonBuilder().create();
String json = NBIO.classpath().name("test.json").one().asString();
String json = NBIO.classpath().pathname("test.json").one().asString();
Type type = new TypeToken<PromQueryResult<PMatrixData>>() {
}.getType();
Object result = gson.fromJson(json, type);

6
engine-core/src/main/java/io/nosqlbench/engine/core/lifecycle/activity/ActivityTypeLoader.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -114,8 +114,8 @@ public class ActivityTypeLoader {
private Set<URL> addLibDir(Set<URL> urlsToAdd, Path libpath) {
Set<URL> urls = NBIO.local()
.prefix(libpath.toString())
.extension(".jar")
.searchPrefixes(libpath.toString())
.extensionSet(".jar")
.list().stream().map(Content::getURL)
.collect(Collectors.toSet());
urlsToAdd.addAll(urls);

10
engine-core/src/main/java/io/nosqlbench/engine/core/metadata/MarkdownFinder.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -53,10 +53,10 @@ public class MarkdownFinder {
public String forResourceMarkdown(String s, String... additionalSearchPaths) {
Optional<Content<?>> docs = NBIO.local()
.prefix("docs")
.prefix(additionalSearchPaths)
.name(s)
.extension(".md")
.searchPrefixes("docs")
.searchPrefixes(additionalSearchPaths)
.pathname(s)
.extensionSet(".md")
.first();
return docs.map(Content::asString).orElse(null);

24
engine-docker/src/main/java/io/nosqlbench/engine/docker/DockerMetricsManager.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -82,19 +82,19 @@ public class DockerMetricsManager {
String GRAFANA_IMG = "grafana/grafana";
tag = (tag == null || tag.isEmpty()) ? "latest" : tag;
String name = "grafana";
List<Integer> port = Arrays.asList(3000);
List<Integer> port = List.of(3000);
boolean grafanaFilesExist = grafanaFilesExist();
if (!grafanaFilesExist) {
setupGrafanaFiles(ip);
}
List<String> volumeDescList = Arrays.asList(
userHome + "/.nosqlbench/grafana:/var/lib/grafana:rw"
//cwd+"/docker-metrics/grafana:/grafana",
//cwd+"/docker-metrics/grafana/datasources:/etc/grafana/provisioning/datasources",
//cwd+"/docker-metrics/grafana/dashboardconf:/etc/grafana/provisioning/dashboards"
//,cwd+"/docker-metrics/grafana/dashboards:/var/lib/grafana/dashboards:ro"
List<String> volumeDescList = List.of(
userHome + "/.nosqlbench/grafana:/var/lib/grafana:rw"
//cwd+"/docker-metrics/grafana:/grafana",
//cwd+"/docker-metrics/grafana/datasources:/etc/grafana/provisioning/datasources",
//cwd+"/docker-metrics/grafana/dashboardconf:/etc/grafana/provisioning/dashboards"
//,cwd+"/docker-metrics/grafana/dashboards:/var/lib/grafana/dashboards:ro"
);
List<String> envList = Arrays.asList(
"GF_SECURITY_ADMIN_PASSWORD=admin",
@ -125,7 +125,7 @@ public class DockerMetricsManager {
logger.info("preparing to start docker metrics");
String PROMETHEUS_IMG = "prom/prometheus";
String name = "prom";
List<Integer> port = Arrays.asList(9090);
List<Integer> port = List.of(9090);
if (!promFilesExist()) {
setupPromFiles(ip);
@ -175,7 +175,7 @@ public class DockerMetricsManager {
setupGraphiteFiles(volumeDescList);
List<String> envList = Arrays.asList();
List<String> envList = List.of();
String reload = null;
List<String> linkNames = new ArrayList();
@ -342,7 +342,7 @@ public class DockerMetricsManager {
private void configureGrafana() {
List<Content<?>> dashboardContent = NBIO.all().prefix("docker/dashboards").extension(".json").list();
List<Content<?>> dashboardContent = NBIO.all().searchPrefixes("docker/dashboards").extensionSet(".json").list();
for (Content<?> content : dashboardContent) {
String dashboardData = content.asString();
@ -355,7 +355,7 @@ public class DockerMetricsManager {
}
List<Content<?>> datasources = NBIO.all().prefix("docker/datasources").extension(".yaml").list();
List<Content<?>> datasources = NBIO.all().searchPrefixes("docker/datasources").extensionSet(".yaml").list();
for (Content<?> datasource : datasources) {
String datasourceContent = datasource.asString();

2
engine-rest/pom.xml
View File

@ -39,7 +39,7 @@
<dependency>
<groupId>io.swagger.parser.v3</groupId>
<artifactId>swagger-parser</artifactId>
<version>2.1.10</version>
<version>2.1.11</version>
</dependency>
<dependency>

14
mvn-defaults/pom.xml
View File

@ -94,19 +94,19 @@
<dependency>
<groupId>info.picocli</groupId>
<artifactId>picocli</artifactId>
<version>4.7.0</version>
<version>4.7.1</version>
</dependency>
<!-- due to https://github.com/oshi/oshi/issues/1289 -->
<dependency>
<groupId>net.java.dev.jna</groupId>
<artifactId>jna</artifactId>
<version>5.12.1</version>
<version>5.13.0</version>
</dependency>
<dependency>
<groupId>net.java.dev.jna</groupId>
<artifactId>jna-platform</artifactId>
<version>5.12.1</version>
<version>5.13.0</version>
</dependency>
<dependency>
@ -300,7 +300,7 @@
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk-s3</artifactId>
<version>1.12.393</version>
<version>1.12.407</version>
</dependency>
<dependency>
@ -336,7 +336,7 @@
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-csv</artifactId>
<version>1.9.0</version>
<version>1.10.0</version>
</dependency>
<dependency>
<groupId>commons-codec</groupId>
@ -383,7 +383,7 @@
<dependency>
<groupId>org.graalvm.sdk</groupId>
<artifactId>graal-sdk</artifactId>
<version>22.3.0</version>
<version>22.3.1</version>
</dependency>
<dependency>
<groupId>org.graalvm.js</groupId>
@ -399,7 +399,7 @@
<dependency>
<groupId>org.graalvm.tools</groupId>
<artifactId>profiler</artifactId>
<version>22.3.0</version>
<version>22.3.1</version>
<scope>runtime</scope>
</dependency>
<dependency>

4
nb-api/src/main/java/io/nosqlbench/api/apps/BundledApp.java
View File

@ -38,10 +38,10 @@ public interface BundledApp extends ToIntFunction<String[]> {
String dev_docspath = "app-" + this.getBundledAppName() + "/src/main/resources/docs/" + this.getBundledAppName();
String cp_docspath = "docs/" + this.getBundledAppName();
Optional<Content<?>> bundled_docs = NBIO.local().name(dev_docspath, cp_docspath).first();
Optional<Content<?>> bundled_docs = NBIO.local().pathname(dev_docspath, cp_docspath).first();
bundled_docs.map(Content::asPath).ifPresent(docs::addContentsOf);
Optional<Content<?>> maindoc = NBIO.local().name("/src/main/resources/" + this.getBundledAppName() + ".md", this.getBundledAppName() + ".md").first();
Optional<Content<?>> maindoc = NBIO.local().pathname("/src/main/resources/" + this.getBundledAppName() + ".md", this.getBundledAppName() + ".md").first();
maindoc.map(Content::asPath).ifPresent(docs::addPath);

4
nb-api/src/main/java/io/nosqlbench/api/config/params/DataSources.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -45,7 +45,7 @@ public class DataSources {
String filename = data.substring("IMPORT{".length(), data.length() - 1);
Path filepath = Path.of(filename);
src = NBIO.all().name(filename).first()
src = NBIO.all().pathname(filename).first()
.map(c -> {
logger.debug(() -> "found 'data' at " + c.getURI());
return c.asString();

7
nb-api/src/main/java/io/nosqlbench/api/config/standard/ConfigLoader.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -103,7 +103,7 @@ public class ConfigLoader {
String filename = data.substring("IMPORT{".length(), data.length() - 1);
Path filepath = Path.of(filename);
data = NBIO.all().name(filename).first()
data = NBIO.all().pathname(filename).first()
.map(c -> {
logger.debug(() -> "found 'data' at " + c.getURI());
return c.asString();
@ -111,9 +111,8 @@ public class ConfigLoader {
}
if (data.startsWith("{") || data.startsWith("[")) {
JsonParser parser = new JsonParser();
JsonElement jsonElement = parser.parse(data);
JsonElement jsonElement = JsonParser.parseString(data);
if (jsonElement.isJsonArray()) {
JsonArray asJsonArray = jsonElement.getAsJsonArray();
for (JsonElement element : asJsonArray) {

297
nb-api/src/main/java/io/nosqlbench/api/content/NBIO.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -31,15 +31,13 @@ import java.util.regex.Pattern;
import java.util.stream.Collectors;
/**
* NBIO is a helper utility packaged as a search builder and fluent API.
* It uses value semantics internally, so it is safe to re-use as a
* stateful configuration builder for finding files in various ways.
* NBIO is a helper utility packaged as a search builder and fluent API. It uses value semantics internally, so it is
* safe to re-use as a stateful configuration builder for finding files in various ways.
* <p>
* Since this is meant to ease development around a usually over-complicated
* surface area in the JVM (Files, Paths, URIs, accessing data, knowing where it comes
* from, searching for it, etc), more emphasis was put on ease of use and
* clarity than efficiency. This set of classes is not expected to be used
* much in NoSqlBench after workload initialization, so is not performance oriented
* Since this is meant to ease development around a usually over-complicated surface area in the JVM (Files, Paths,
* URIs, accessing data, knowing where it comes from, searching for it, etc), more emphasis was put on ease of use and
* clarity than efficiency. This set of classes is not expected to be used much in NoSqlBench after workload
* initialization, so is not performance oriented.
*/
public class NBIO implements NBPathsAPI.Facets {
@ -52,7 +50,8 @@ public class NBIO implements NBPathsAPI.Facets {
private URIResolver resolver;
private List<String> names = new ArrayList<>();
private List<String> extensions = new ArrayList<>();
// private List<String> extensions = new ArrayList<>();
private List<Set<String>> extensionSets = new ArrayList<>();
private Set<String> prefixes = new HashSet<>(Arrays.asList(globalIncludes));
private NBIO() {
@ -61,15 +60,15 @@ public class NBIO implements NBPathsAPI.Facets {
private NBIO(URIResolver resolver,
Set<String> prefixes,
List<String> names,
List<String> extensions) {
List<Set<String>> extensionSets) {
this.resolver = resolver;
this.prefixes = prefixes;
this.names = names;
this.extensions = extensions;
this.extensionSets = extensionSets;
}
public static List<String> readLines(String filename) {
Content<?> data = NBIO.all().prefix("data").name(filename).first().orElseThrow(
Content<?> data = NBIO.all().searchPrefixes("data").pathname(filename).first().orElseThrow(
() -> new BasicError("Unable to read lines from " + filename)
);
String[] split = data.getCharBuffer().toString().split("\n");
@ -94,26 +93,26 @@ public class NBIO implements NBPathsAPI.Facets {
private static InputStream readInputStream(String filename, String... searchPaths) {
return NBIO.all().prefix(searchPaths).name(filename).one().getInputStream();
return NBIO.all().searchPrefixes(searchPaths).pathname(filename).one().getInputStream();
}
private static Reader readReader(String filename, String... searchPaths) {
return NBIO.all().prefix(searchPaths).name(filename).one().getReader();
return NBIO.all().searchPrefixes(searchPaths).pathname(filename).one().getReader();
}
public static CharBuffer readCharBuffer(String fileName, String... searchPaths) {
return NBIO.all().prefix(searchPaths).name(fileName).one().getCharBuffer();
return NBIO.all().searchPrefixes(searchPaths).pathname(fileName).one().getCharBuffer();
}
public static Path getFirstLocalPath(String... potentials) {
Optional<Content<?>> first = NBIO.local().name(potentials).first();
Optional<Content<?>> first = NBIO.local().pathname(potentials).first();
return first.orElseThrow(
() -> new BasicError("Unable to find loadable content at " + String.join(",", potentials))
).asPath();
}
public static Optional<Path> findFirstLocalPath(String... potentials) {
Optional<Content<?>> first = NBIO.local().name(potentials).first();
Optional<Content<?>> first = NBIO.local().pathname(potentials).first();
Optional<Path> path = first.map(Content::asPath);
return path;
}
@ -123,46 +122,67 @@ public class NBIO implements NBPathsAPI.Facets {
}
/**
* {@inheritDoc}
*/
@Override
public NBPathsAPI.GetPrefix localContent() {
public NBPathsAPI.GetPrefixes localContent() {
this.resolver = URIResolvers.inFS().inCP();
return this;
}
/**
* {@inheritDoc}
*/
@Override
public NBPathsAPI.GetPrefix remoteContent() {
public NBPathsAPI.GetPrefixes remoteContent() {
this.resolver = URIResolvers.inURLs();
return this;
}
/**
* {@inheritDoc}
*/
@Override
public NBPathsAPI.GetPrefix internalContent() {
public NBPathsAPI.GetPrefixes internalContent() {
this.resolver = URIResolvers.inClasspath();
return this;
}
/**
* {@inheritDoc}
*/
@Override
public NBPathsAPI.GetPrefix fileContent() {
public NBPathsAPI.GetPrefixes fileContent() {
this.resolver = URIResolvers.inFS();
return this;
}
/**
* {@inheritDoc}
*/
@Override
public NBPathsAPI.GetPrefix allContent() {
public NBPathsAPI.GetPrefixes allContent() {
this.resolver = URIResolvers.inFS().inCP().inURLs();
return this;
}
/**
* {@inheritDoc}
*/
@Override
public NBPathsAPI.GetPrefix prefix(String... searchPaths) {
public NBPathsAPI.GetPrefixes searchPrefixes(String... searchPaths) {
HashSet<String> addingPaths = new HashSet<>(this.prefixes);
addingPaths.addAll(Arrays.asList(searchPaths));
return new NBIO(resolver, addingPaths, names, extensions);
return new NBIO(resolver, addingPaths, names, extensionSets);
}
private final static Pattern extensionPattern = Pattern.compile("\\.[a-zA-Z]+");
private final static Pattern wildcardsPattern = Pattern.compile(".*?[^?+*][\\?\\+\\*].*");
/**
* {@inheritDoc}
*/
@Override
public NBPathsAPI.DoSearch search(String... searches) {
List<String> prefixesToAdd = new ArrayList<>();
@ -185,7 +205,7 @@ public class NBIO implements NBPathsAPI.Facets {
int literalsTill = 0;
while (true) {
if (literalsTill>= parts.length-1) {
if (literalsTill >= parts.length - 1) {
break;
}
if (wildcardsPattern.matcher(parts[literalsTill]).matches()) {
@ -201,31 +221,81 @@ public class NBIO implements NBPathsAPI.Facets {
prefixesToAdd.add(prefix);
String[] nameary = new String[parts.length - literalsTill];
System.arraycopy(parts,literalsTill,nameary,0,nameary.length);
System.arraycopy(parts, literalsTill, nameary, 0, nameary.length);
String name = String.join(File.separator, nameary);
namesToAdd.add(name);
}
return prefix(prefixesToAdd.toArray(new String[]{}))
.name(namesToAdd.toArray(new String[]{}))
.extension(extensionsToAdd.toArray(new String[]{}));
return searchPrefixes(prefixesToAdd.toArray(new String[]{}))
.pathname(namesToAdd.toArray(new String[]{}))
.extensionSet(extensionsToAdd.toArray(new String[]{}));
}
/**
* {@inheritDoc}
*/
@Override
public NBPathsAPI.GetExtension name(String... searchNames) {
public NBPathsAPI.GetExtensions pathname(String... searchNames) {
ArrayList<String> addingNames = new ArrayList<>(this.names);
addingNames.addAll(Arrays.asList(searchNames));
return new NBIO(resolver, prefixes, addingNames, extensions);
return new NBIO(resolver, prefixes, addingNames, extensionSets);
}
/**
* {@inheritDoc}
*/
@Override
public NBPathsAPI.DoSearch extension(String... extensions) {
ArrayList<String> addingExtensions = new ArrayList<>(this.extensions);
for (String addingExtension : extensions) {
addingExtensions.add(dotExtension(addingExtension));
public NBPathsAPI.GetExtensions extensionSet(String... extensions) {
if (extensions.length==0) {
return this;
}
return new NBIO(resolver, prefixes, names, addingExtensions);
return this.extensionSets(
new HashSet<>(
Arrays.asList(extensions).stream()
.map(String::toLowerCase)
.map(s -> s.startsWith(".") ? s : "." + s)
.collect(Collectors.toList())
)
);
}
/**
* {@inheritDoc}
*/
@Override
public NBPathsAPI.GetExtensions extensionSets(Set<String>... cosets) {
for (Set<String> coset : cosets) {
if (coset.size()==0) {
throw new RuntimeException("Extension co-set can not be empty.");
}
for (String entry : coset) {
String adjusted = (entry.startsWith(".") ? entry : "." + entry).toLowerCase();
if (coset.add(adjusted)) {
coset.remove(entry);
}
}
Set<String> addTo = null;
for (Set<String> extensionSet : this.extensionSets) {
Set<String> union = new LinkedHashSet<>(coset);
for (String entry : coset) {
if (extensionSet.contains(entry)) {
addTo = extensionSet;
break;
}
}
if (addTo != null) {
break;
}
}
if (addTo == null) {
addTo = new LinkedHashSet<>();
extensionSets.add(addTo);
}
addTo.addAll(coset);
}
return this;
}
/**
@ -233,7 +303,7 @@ public class NBIO implements NBPathsAPI.Facets {
*
* @return a builder
*/
public static NBPathsAPI.GetPrefix all() {
public static NBPathsAPI.GetPrefixes all() {
return new NBIO().allContent();
}
@ -242,7 +312,7 @@ public class NBIO implements NBPathsAPI.Facets {
*
* @return a builder
*/
public static NBPathsAPI.GetPrefix classpath() {
public static NBPathsAPI.GetPrefixes classpath() {
return new NBIO().internalContent();
}
@ -251,7 +321,7 @@ public class NBIO implements NBPathsAPI.Facets {
*
* @return a builder
*/
public static NBPathsAPI.GetPrefix fs() {
public static NBPathsAPI.GetPrefixes fs() {
return new NBIO().fileContent();
}
@ -260,7 +330,7 @@ public class NBIO implements NBPathsAPI.Facets {
*
* @return a builder
*/
public static NBPathsAPI.GetPrefix local() {
public static NBPathsAPI.GetPrefixes local() {
return new NBIO().localContent();
}
@ -269,11 +339,14 @@ public class NBIO implements NBPathsAPI.Facets {
*
* @return a builder
*/
public static NBPathsAPI.GetPrefix remote() {
public static NBPathsAPI.GetPrefixes remote() {
return new NBIO().remoteContent();
}
/**
* {@inheritDoc}
*/
@Override
public Optional<Content<?>> first() {
@ -296,9 +369,38 @@ public class NBIO implements NBPathsAPI.Facets {
throw new RuntimeException("Invalid code, go fix it, this should never happen.");
}
/**
* {@inheritDoc}
*/
@Override
public Content<?> one() {
if (extensionSets.size()==0) {
for (String name : names) {
Content<?> found = resolver.resolveOne(name);
if (found != null) {
return found;
}
}
} else {
for (String name : names) {
Content<?> found = resolver.resolveOne(name);
if (found != null) {
return found;
}
for (Set<String> extensionSet : extensionSets) {
for (String es : extensionSet) {
for (String extension : extensionSet) {
found = resolver.resolveOne(name + extension);
if (found != null) {
return found;
}
}
}
}
}
}
List<Content<?>> list = list();
if (list.size() == 0) {
@ -313,11 +415,14 @@ public class NBIO implements NBPathsAPI.Facets {
}
/**
* {@inheritDoc}
*/
@Override
public List<List<Content<?>>> resolveEach() {
List<List<Content<?>>> resolved = new ArrayList<>();
for (String name : names) {
LinkedHashSet<String> slotSearchPaths = expandNamesAndSuffixes(List.of(name), new HashSet<>(extensions));
LinkedHashSet<String> slotSearchPaths = expandNamesAndSuffixes(this.prefixes, List.of(name), extensionSets);
Content<?> content = null;
for (String slotSearchPath : slotSearchPaths) {
List<Content<?>> contents = resolver.resolve(slotSearchPath);
@ -330,7 +435,7 @@ public class NBIO implements NBPathsAPI.Facets {
// for testing
public LinkedHashSet<String> expandNamesAndSuffixes() {
return expandNamesAndSuffixes(names, new HashSet(extensions));
return expandNamesAndSuffixes(prefixes, names, extensionSets);
}
@ -339,46 +444,92 @@ public class NBIO implements NBPathsAPI.Facets {
* If no name is given, then <pre>{@code .*}</pre> is used.
* If suffixes are given, then all returned results must include at least
* one of the suffixes. If the name includes one of the suffixes given,
* then additional names are expanded to match the additional suffixes.
* then additional suffixes are added to the list of searchable path names.
*
* @param _names base filenames or path fragment, possibly fully-qualified
* @param _suffixes zero or more suffixes, which, if given, imply that one of them must match
* @param _names
* base filenames or path fragment, possibly fully-qualified
* @param _suffixCosetLists
* zero or more suffixes, which, if given, imply that one of them must match
* @return Expanded names of valid filename fragments according to the above rules
*/
public LinkedHashSet<String> expandNamesAndSuffixes(
public static LinkedHashSet<String> expandNamesAndSuffixes(
Set<String> _prefixes,
List<String> _names,
Set<String> _suffixes) {
List<Set<String>> _suffixCosetLists) {
LinkedHashSet<String> searches = new LinkedHashSet<>();
if (_names.size() == 0 && prefixes.size() == 0) {
searches.add(".*");
} else if (_names.size() > 0 && _suffixes.size() == 0) {
searches.addAll(_names);
} else if (_names.size() == 0 && _suffixes.size() > 0) {
_suffixes.stream().map(s -> ".*" + s).forEach(searches::add);
if (_names.size() > 0) {
if (_suffixCosetLists.size() > 0) { // names & suffixes
searches.addAll(expandSynonymPaths(_names, _suffixCosetLists));
} else { // names only
searches.addAll(_names);
}
} else {
for (String name : _names) {
// if (!name.equals(".*")) {
// searches.add(name);
// }
String basename = name;
boolean suffixed = false;
for (String suffix : _suffixes) {
if (name.endsWith(suffix)) {
suffixed = true;
basename = name.substring(0, name.length() - suffix.length());
break;
if (_suffixCosetLists.size() > 0) { // suffixes only
for (Set<String> suffixCoset : _suffixCosetLists) {
for (String suffix : suffixCoset) {
searches.add(".*" + suffix);
}
}
for (String suffix : _suffixes) {
searches.add(basename + suffix);
}
} else { // neither
searches.add(".*");
}
}
if (_prefixes.size() > 0) {
LinkedHashSet<String> prefixed = new LinkedHashSet<>(searches);
for (String prefix : _prefixes) {
for (String search : searches) {
prefixed.add(prefix + File.separator + search);
}
}
searches = prefixed;
}
return searches;
}
public static Set<String> expandSynonymPaths(List<String> names, List<Set<String>> suffixSets) {
if (suffixSets.size()==0) {
return new LinkedHashSet<>(names);
}
Set<String> newnames = new LinkedHashSet<>();
for (String name : names) {
Set<String> matchingSet = null;
String basename = null;
for (Set<String> suffixSet : suffixSets) {
for (String suffix : suffixSet) {
if (name.endsWith(suffix)) {
matchingSet = suffixSet;
basename = name.substring(0, name.length() - suffix.length());
// newnames.add(name); // Leave this here, it initializes precedence
break;
}
}
if (basename != null) {
break;
}
}
if (basename == null) {
if (name.contains(".") && !name.equals(".*")) {
// newnames.add(name);
suffixSets.stream().flatMap(s -> s.stream()).map(s -> name + s).forEach(newnames::add);
} else {
suffixSets.stream().flatMap(s -> s.stream()).map(s -> name + s).forEach(newnames::add);
}
} else {
for (String extension : matchingSet) {
newnames.add(basename + extension);
}
}
}
return newnames;
}
/**
* {@inheritDoc}
*/
@Override
public List<Content<?>> list() {
LinkedHashSet<String> searches = expandNamesAndSuffixes();
@ -413,6 +564,9 @@ public class NBIO implements NBPathsAPI.Facets {
return new ArrayList<>(foundFiles);
}
/**
* {@inheritDoc}
*/
@Override
public List<Path> relativeTo(String... base) {
String base1 = base[0];
@ -462,13 +616,16 @@ public class NBIO implements NBPathsAPI.Facets {
// }
/**
* {@inheritDoc}
*/
@Override
public String toString() {
return "NBIO{" +
"resolver=" + resolver +
", prefixes=" + prefixes +
", names=" + names +
", extensions=" + extensions +
", extensionSets=" + extensionSets +
'}';
}
}

38
nb-api/src/main/java/io/nosqlbench/api/content/NBPathsAPI.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -19,11 +19,21 @@ package io.nosqlbench.api.content;
import java.nio.file.Path;
import java.util.List;
import java.util.Optional;
import java.util.Set;
/**
* This API provides a single service to locate and load content resource from multiple places,
* at build time and runtime, using a standard fluent-like pattern. The most important details for new uesrs are:
* <UL>
* <LI>Search paths are optional, and are tried <strong>after</strong> non-prefixed paths.</LI>
* <LI>If extensions are provided, then any result returned <strong>must</strong> match one of the extensions.</LI>
* <LI>The most efficient way to access a single path will be to use the {@link Facets#one()} method as {@link NBIO#one()}</LI>
* </UL>
*/
public interface NBPathsAPI {
interface Facets extends
GetSource, GetPrefix, GetName, GetExtension, DoSearch {
GetSource, GetPrefixes, GetPathname, GetExtensions, DoSearch {
}
interface GetSource {
@ -32,7 +42,7 @@ public interface NBPathsAPI {
*
* @return this builder
*/
GetPrefix localContent();
GetPrefixes localContent();
/**
* Only return content from remote URLs. If the user is providing non-URL content
@ -40,7 +50,7 @@ public interface NBPathsAPI {
*
* @return this builder
*/
GetPrefix remoteContent();
GetPrefixes remoteContent();
/**
* Only return content from the runtime classpath, internal resources that are bundled,
@ -48,14 +58,14 @@ public interface NBPathsAPI {
*
* @return this builder
*/
GetPrefix internalContent();
GetPrefixes internalContent();
/**
* Only return content from the filesystem, but not remote URLs nor internal bundled resources.
*
* @return this builder
*/
GetPrefix fileContent();
GetPrefixes fileContent();
/**
* Return content from everywhere, from remote URls, or from the file system and then the internal
@ -63,10 +73,10 @@ public interface NBPathsAPI {
*
* @return this builder
*/
GetPrefix allContent();
GetPrefixes allContent();
}
interface GetPrefix extends GetName {
interface GetPrefixes extends GetPathname {
/**
* Each of the prefix paths will be searched if the resource is not found with the exact
* path given. To be specific, if you want to search within a location based on wildcards,
@ -75,10 +85,10 @@ public interface NBPathsAPI {
* @param prefixPaths A list of paths to include in the search
* @return this builder
*/
GetPrefix prefix(String... prefixPaths);
GetPrefixes searchPrefixes(String... prefixPaths);
}
interface GetName extends GetExtension {
interface GetPathname extends GetExtensions {
/**
* Provide the names of the resources to be resolved. More than one resource may be provided.
* If no name is provided, then a wildcard search is assumed.
@ -86,7 +96,7 @@ public interface NBPathsAPI {
* @param name The name of the resource to load
* @return this builder
*/
GetExtension name(String... name);
GetExtensions pathname(String... name);
/**
* Provide a combined prefix, name and suffix in a combined form. For each search template provided,
@ -123,7 +133,7 @@ public interface NBPathsAPI {
}
interface GetExtension extends DoSearch {
interface GetExtensions extends DoSearch {
/**
* provide a list of optional file extensions which should be considered. If the content is
* not found under the provided name, then each of the extension is tried in order.
@ -134,8 +144,8 @@ public interface NBPathsAPI {
* @param extensions The extension names to try
* @return this builder
*/
DoSearch extension(String... extensions);
GetExtensions extensionSet(String... extensions);
GetExtensions extensionSets(Set<String>... cosets);
}
interface DoSearch {

29
nb-api/src/main/java/io/nosqlbench/api/content/URIResolver.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -22,6 +22,7 @@ import java.net.URI;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
/**
* This is a stateful search object for resources like Paths or URLs.
@ -48,6 +49,7 @@ public class URIResolver implements ContentResolver {
* Include resources from all known places, including remote URLs,
* the local default filesystem, and the classpath, which includes
* the jars that hold the current runtime application.
*
* @return this URISearch
*/
public URIResolver all() {
@ -57,6 +59,7 @@ public class URIResolver implements ContentResolver {
/**
* Includ resources in the default filesystem
*
* @return this URISearch
*/
public URIResolver inFS() {
@ -66,6 +69,7 @@ public class URIResolver implements ContentResolver {
/**
* Include resources in remote URLs
*
* @return this URISearch
*/
public URIResolver inURLs() {
@ -75,6 +79,7 @@ public class URIResolver implements ContentResolver {
/**
* Include resources within the classpath.
*
* @return this URISearch
*/
public URIResolver inCP() {
@ -117,26 +122,38 @@ public class URIResolver implements ContentResolver {
}
public URIResolver extension(String extension) {
this.extensions = this.extensions==null ? new ArrayList<>() : this.extensions;
this.extensions = this.extensions == null ? new ArrayList<>() : this.extensions;
this.extensions.add(extension);
return this;
}
public URIResolver extraPaths(String extraPath) {
this.extraPaths = this.extraPaths==null ? new ArrayList<>() : this.extraPaths;
this.extraPaths = this.extraPaths == null ? new ArrayList<>() : this.extraPaths;
this.extraPaths.add(Path.of(extraPath));
return this;
}
public Optional<Content<?>> resolveOneOptionally(String candidatePath) {
List<Content<?>> contents = resolveAll(candidatePath);
if (contents.size() == 1) {
return Optional.of(contents.get(0));
}
if (contents.size() == 0) {
return Optional.empty();
}
throw new BasicError("Error while loading content '" + candidatePath + "', only one is allowed, but " + contents.size() + " were found");
}
public Content<?> resolveOne(String candidatePath) {
List<Content<?>> contents = resolveAll(candidatePath);
if (contents.size()==1) {
if (contents.size() == 1) {
return contents.get(0);
}
if (contents.size()==0) {
if (contents.size() == 0) {
return null;
}
throw new BasicError("Error while loading content '" + candidatePath +"', only one is allowed, but " + contents.size() + " were found");
throw new BasicError("Error while loading content '" + candidatePath + "', only one is allowed, but " + contents.size() + " were found");
}
public String toString() {

4
nb-api/src/main/java/io/nosqlbench/api/markdown/providers/DocsRootDirectory.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -29,7 +29,7 @@ public abstract class DocsRootDirectory implements RawMarkdownSource {
@Override
public List<Content<?>> getMarkdownInfo() {
List<Content<?>> list = NBIO.local().name(getRootPathName()).list();
List<Content<?>> list = NBIO.local().pathname(getRootPathName()).list();
NBIOWalker.CollectVisitor capture = new NBIOWalker.CollectVisitor(true, false);
NBIOWalker.RegexFilter filter = new NBIOWalker.RegexFilter("\\.md",true);
for (Content<?> content : list) {

25
nb-api/src/test/java/io/nosqlbench/nb/api/config/ConfigModelTest.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -16,9 +16,7 @@
package io.nosqlbench.nb.api.config;
import io.nosqlbench.api.config.standard.ConfigModel;
import io.nosqlbench.api.config.standard.NBConfiguration;
import io.nosqlbench.api.config.standard.Param;
import io.nosqlbench.api.config.standard.*;
import org.junit.jupiter.api.Test;
import java.util.List;
@ -36,6 +34,25 @@ public class ConfigModelTest {
NBConfiguration cfg = cm.apply(Map.of("c", 232));
assertThat(cfg.getOptional("a")).isEmpty();
assertThat(cfg.get("c",int.class)).isEqualTo(232);
}
@Test
public void testBoxingSupport() {
NBConfigModel model = ConfigModel.of(ConfigModelTest.class)
.add(Param.defaultTo("val",5))
.asReadOnly();
NBConfiguration config = model.apply(Map.of("val", 7));
Integer val1 = config.getOrDefault("val", 8);
assertThat(val1).isEqualTo(7);
int val2 = config.getOrDefault("val", 9);
assertThat(val2).isEqualTo(7);
Integer val3 = config.get("val");
assertThat(val3).isEqualTo(7);
int val4 = config.get("val");
assertThat(val4).isEqualTo(7);
}
}

161
nb-api/src/test/java/io/nosqlbench/nb/api/content/NBIOTest.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -26,13 +26,13 @@ import org.junit.jupiter.api.Test;
import java.io.File;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
import static org.assertj.core.api.Assertions.assertThat;
@ -42,72 +42,80 @@ public class NBIOTest {
@Test
public void testFullyQualifiedNameSearches() {
NBIO extensions = (NBIO) NBIO.all().name("foo.bar");
NBIO extensions = (NBIO) NBIO.all().pathname("foo.bar");
LinkedHashSet<String> searches = extensions.expandNamesAndSuffixes();
assertThat(searches).containsExactly("foo.bar");
}
@Test
public void testExpandWildcardAndExtensionsOnly() {
NBIO extensions = (NBIO) NBIO.all().name(".*").extension("foo","bar");
NBIO extensions = (NBIO) NBIO.all().pathname(".*").extensionSet("foo","bar");
LinkedHashSet<String> searches = extensions.expandNamesAndSuffixes();
assertThat(searches).contains(".*.foo",".*.bar");
}
@Test
public void testExpandNameAndAllSuffixesOnly() {
NBIO extensions = (NBIO) NBIO.all().name("foo.bar").extension("test1","test2");
NBIO extensions = (NBIO) NBIO.all().pathname("foo.bar").extensionSet("test1","test2");
LinkedHashSet<String> searches = extensions.expandNamesAndSuffixes();
assertThat(searches).containsExactly("foo.bar.test1","foo.bar.test2");
}
@Test
public void testExpandNamesAndExtensionsIfNotExtended() {
NBIO extensions = (NBIO) NBIO.all().name("foo").extension("baz","beez");
NBIO extensions = (NBIO) NBIO.all().pathname("foo").extensionSet("baz","beez");
LinkedHashSet<String> searches = extensions.expandNamesAndSuffixes();
assertThat(searches).contains("foo.baz","foo.beez");
}
@Test
public void testExpandNamesAndExtensionsAvoidsExtendedAlreadyExtended() {
NBIO extensions = (NBIO) NBIO.all().name("foo.baz").extension("baz","beez");
NBIO extensions = (NBIO) NBIO.all().pathname("foo.baz").extensionSet("baz","beez");
LinkedHashSet<String> searches = extensions.expandNamesAndSuffixes();
assertThat(searches).contains("foo.baz","foo.beez");
}
@Test
public void testExpandPrefixesAndFullName() {
NBIO extensions = (NBIO) NBIO.all().prefix("act1","act2").name("foo.bar");
NBIO extensions = (NBIO) NBIO.all().searchPrefixes("act1","act2").pathname("foo.bar");
LinkedHashSet<String> searches = extensions.expandNamesAndSuffixes();
assertThat(searches).containsExactly("foo.bar");
assertThat(searches).containsExactly("foo.bar","act1/foo.bar","act2/foo.bar");
}
@Test
public void testExpandAddExtensionNotNeeded() {
NBIO extensions = (NBIO) NBIO.all().name("foo.bar").extension("bar");
NBIO extensions = (NBIO) NBIO.all().pathname("foo.bar").extensionSet("bar");
LinkedHashSet<String> searches = extensions.expandNamesAndSuffixes();
assertThat(searches).containsExactly("foo.bar");
NBIO extensionsDot = (NBIO) NBIO.all().name("foo.bar").extension(".bar");
NBIO extensionsDot = (NBIO) NBIO.all().pathname("foo.bar").extensionSet(".bar");
LinkedHashSet<String> searchesDot = extensionsDot.expandNamesAndSuffixes();
assertThat(searchesDot).containsExactly("foo.bar");
}
@Test
public void testExpandExtensionCosets() {
Set<String> paths = NBIO.expandSynonymPaths(List.of("abc123.tubas"), List.of(Set.of(".foo")));
assertThat(paths).isEqualTo(Set.of("abc123.tubas.foo"));
paths = NBIO.expandSynonymPaths(List.of("abc123.tubas","def.123"), List.of(Set.of(".456",".789",".123")));
assertThat(paths).isEqualTo(Set.of("abc123.tubas.123","abc123.tubas.456","abc123.tubas.789","def.123","def.456","def.789"));
}
@Test
public void testExpandAddExtensionNeeded() {
NBIO extensions = (NBIO) NBIO.all().name("foo").extension("bar");
NBIO extensions = (NBIO) NBIO.all().pathname("foo").extensionSet("bar");
LinkedHashSet<String> searches = extensions.expandNamesAndSuffixes();
assertThat(searches).containsExactly("foo.bar");
NBIO extensionsDot = (NBIO) NBIO.all().name("foo").extension(".bar");
NBIO extensionsDot = (NBIO) NBIO.all().pathname("foo").extensionSet(".bar");
LinkedHashSet<String> searchesDot = extensionsDot.expandNamesAndSuffixes();
assertThat(searchesDot).containsExactly("foo.bar");
}
@Test
public void testLoadCsv1Classpath() {
NBPathsAPI.GetPrefix forSourceType = NBIO.classpath();
NBPathsAPI.GetName nesteddir1 = forSourceType.prefix("nesteddir1");
NBPathsAPI.GetExtension getExtension = nesteddir1.name("nesteddir2/testcsv1");
NBPathsAPI.DoSearch forCsvExtension = getExtension.extension(".csv");
NBPathsAPI.GetPrefixes forSourceType = NBIO.classpath();
NBPathsAPI.GetPathname nesteddir1 = forSourceType.searchPrefixes("nesteddir1");
NBPathsAPI.GetExtensions getExtensions = nesteddir1.pathname("nesteddir2/testcsv1");
NBPathsAPI.DoSearch forCsvExtension = getExtensions.extensionSet(".csv");
Optional<Content<?>> testcsv1 = forCsvExtension.first();
assertThat(testcsv1).isNotPresent();
@ -115,10 +123,10 @@ public class NBIOTest {
@Test
public void testLoadCsv1Filesystem() {
NBPathsAPI.GetPrefix forSourceType = NBIO.fs();
NBPathsAPI.GetName nesteddir1 = forSourceType.prefix("target/test-classes/nesteddir1");
NBPathsAPI.GetExtension getExtension = nesteddir1.name("nesteddir2/testcsv1");
NBPathsAPI.DoSearch forCsvExtension = getExtension.extension(".csv");
NBPathsAPI.GetPrefixes forSourceType = NBIO.fs();
NBPathsAPI.GetPathname nesteddir1 = forSourceType.searchPrefixes("target/test-classes/nesteddir1");
NBPathsAPI.GetExtensions getExtensions = nesteddir1.pathname("nesteddir2/testcsv1");
NBPathsAPI.DoSearch forCsvExtension = getExtensions.extensionSet(".csv");
Optional<Content<?>> testcsv1 = forCsvExtension.first();
assertThat(testcsv1).isNotPresent();
@ -130,7 +138,7 @@ public class NBIOTest {
@Test
public void testClasspathTestResource() {
List<List<Content<?>>> optionals =
NBIO.classpath().name("nesteddir1/nesteddir2/testcsv12.csv").resolveEach();
NBIO.classpath().pathname("nesteddir1/nesteddir2/testcsv12.csv").resolveEach();
assertThat(optionals).hasSize(1);
Content<?> content = optionals.get(0).get(0);
assertThat(content).isNotNull();
@ -139,9 +147,9 @@ public class NBIOTest {
@Test
public void testPathSearchForExtension() {
List<Content<?>> list = NBIO.classpath()
.prefix("nesteddir1")
.name(".*.csv")
.extension("csv")
.searchPrefixes("nesteddir1")
.pathname(".*.csv")
.extensionSet("csv")
.list();
assertThat(list).hasSize(2);
}
@ -149,9 +157,9 @@ public class NBIOTest {
@Test
public void testPathSearchForExtensionMissing() {
List<Content<?>> list = NBIO.classpath()
.prefix("nesteddir1")
.name(".*")
.extension("csv")
.searchPrefixes("nesteddir1")
.pathname(".*")
.extensionSet("csv")
.list();
assertThat(list).hasSize(2);
}
@ -159,9 +167,9 @@ public class NBIOTest {
@Test
public void testPathSearchForMultipleExtensions() {
List<Content<?>> list = NBIO.classpath()
.prefix("nesteddir1")
.name(".*")
.extension("csv","txt")
.searchPrefixes("nesteddir1")
.pathname(".*")
.extensionSet("csv","txt")
.list();
assertThat(list).hasSize(3);
}
@ -169,9 +177,9 @@ public class NBIOTest {
@Test
public void testPathSearchForSuffix() {
List<Content<?>> list = NBIO.classpath()
.prefix("nesteddir1")
.name("nesteddir2/testdata12")
.extension("txt")
.searchPrefixes("nesteddir1")
.pathname("nesteddir2/testdata12")
.extensionSet("txt")
.list();
assertThat(list).hasSize(1);
}
@ -179,8 +187,8 @@ public class NBIOTest {
@Test
public void testPathSearchInDifferentVantagePoints() {
List<Path> list = NBIO.fs()
.prefix("target/test-classes/nesteddir1")
.extension("csv")
.searchPrefixes("target/test-classes/nesteddir1")
.extensionSet("csv")
.list().stream().map(Content::asPath)
.collect(Collectors.toList());
@ -193,19 +201,19 @@ public class NBIOTest {
@Test
public void testLoadNamedFileAsYmlExtension() {
List<Content<?>> list = NBIO.classpath()
.name("nesteddir1/nesteddir2/testworkload1")
.extension("yml")
.pathname("nesteddir1/nesteddir2/testworkload1")
.extensionSet("yml")
.list();
assertThat(list).hasSize(1);
list = NBIO.classpath()
.name("nesteddir1/nesteddir2/testworkload1.yml")
.pathname("nesteddir1/nesteddir2/testworkload1.yml")
.list();
assertThat(list).hasSize(1);
list = NBIO.classpath()
.name("nesteddir1/nesteddir2/testworkload1")
.extension("abc","yml")
.pathname("nesteddir1/nesteddir2/testworkload1")
.extensionSet("abc","yml")
.list();
assertThat(list).hasSize(1);
}
@ -214,15 +222,15 @@ public class NBIOTest {
public void testLoadAllFilesUnderPath() {
List<Content<?>> list = null;
list = NBIO.classpath().prefix("./").list();
list = NBIO.classpath().searchPrefixes("./").list();
logger.debug("found " + list.size() + " entries for path '.'");
assertThat(list).hasSizeGreaterThan(0);
list = NBIO.fs().prefix("./").list();
list = NBIO.fs().searchPrefixes("./").list();
logger.debug("found " + list.size() + " entries for path '.'");
assertThat(list).hasSizeGreaterThan(0);
list = NBIO.remote().prefix("./").list();
list = NBIO.remote().searchPrefixes("./").list();
logger.debug("found " + list.size() + " entries for path '.'");
assertThat(list).hasSize(0);
}
@ -230,17 +238,17 @@ public class NBIOTest {
@Test
public void test() {
List<Content<?>> list = NBIO.fs()
.prefix(Paths.get("target/test-classes/").toString())
.name("gamma.yaml").list();
.searchPrefixes(Paths.get("target/test-classes/").toString())
.pathname("gamma.yaml").list();
assertThat(list).hasSize(1);
}
@Test
public void testWildcardFilenameMatch() {
NBPathsAPI.DoSearch gammasSearch = NBIO.all()
.prefix(Paths.get("target/test-classes/").toString())
.name(".*gamma")
.extension("yaml");
.searchPrefixes(Paths.get("target/test-classes/").toString())
.pathname(".*gamma")
.extensionSet("yaml");
List<Content<?>> gammas = gammasSearch.list();
assertThat(gammas).hasSize(3);
}
@ -249,33 +257,57 @@ public class NBIOTest {
@Test
public void testSpecificFilenameMatch() {
NBPathsAPI.DoSearch gammasSearch = NBIO.all()
.prefix(Paths.get("target/test-classes/").toString())
.name("gamma")
.extension("yaml");
.searchPrefixes(Paths.get("target/test-classes/").toString())
.pathname("gamma")
.extensionSet("yaml");
List<Content<?>> gammas = gammasSearch.list();
assertThat(gammas).hasSize(1);
}
@Test
public void onlyMatchExtensionFilesWhenExtensionsProvided() {
public void matchOneWithoutTryingPrefixesFirst() {
Content<?> result = NBIO.all()
.searchPrefixes(
Paths.get("target/test-classes/nesteddir1/nesteddir2").toString()
)
.pathname("nesteddir1/alpha-gamma.yaml")
.one();
assertThat(result).isNotNull();
assertThat(result.getURI().toString()).matches(".*?[^1]/nesteddir1/alpha-gamma.yaml");
}
@Test
public void matchOneFallsThroughToPrefixesSecond() {
Content<?> result = NBIO.all()
.searchPrefixes(
Paths.get("target/test-classes/nesteddir1/nesteddir2").toString()
)
.pathname("alpha-gamma.yaml")
.one();
assertThat(result).isNotNull();
assertThat(result.getURI().toString()).matches(".*?nesteddir1/nesteddir2/alpha-gamma.yaml");
}
@Test
public void onlyMatchExtensionFilesWhenExtensionInCoset() {
// This search is invalid because by providing extensions, all results
// are required to match one of the extensions, thus the only valid
// match here would be alpha-gamma.yaml.js
NBPathsAPI.DoSearch invalidSearch = NBIO.all()
.prefix(Paths.get("target/test-classes/").toString())
.name("alpha-gamma.yaml")
.extension("js");
.searchPrefixes(Paths.get("target/test-classes/").toString())
.pathname("alpha-gamma.yaml")
.extensionSet("js");
NBPathsAPI.DoSearch validSearch1 = NBIO.all()
.prefix(Paths.get("target/test-classes/").toString())
.name("alpha-gamma")
.extension("js");
.searchPrefixes(Paths.get("target/test-classes/").toString())
.pathname("alpha-gamma")
.extensionSet("js");
NBPathsAPI.DoSearch validSearch2 = NBIO.all()
.prefix(Paths.get("target/test-classes/").toString())
.name("alpha-gamma.js")
.extension();
.searchPrefixes(Paths.get("target/test-classes/").toString())
.pathname("alpha-gamma.js")
.extensionSet();
assertThat(invalidSearch.list()).hasSize(0);
@ -292,12 +324,13 @@ public class NBIOTest {
File tempFile = File.createTempFile(tmpdir.toString(), "testfile.csv");
tempFile.deleteOnExit();
String fullpath = tempFile.getAbsolutePath();
Files.write(Path.of(fullpath), "COL1,COL2\n\"val1\",\"val2\"\n".getBytes(StandardCharsets.UTF_8));
List<Content<?>> results = NBIO.all().name(fullpath).list();
Files.writeString(Path.of(fullpath), "COL1,COL2\n\"val1\",\"val2\"\n");
List<Content<?>> results = NBIO.all().pathname(fullpath).list();
assertThat(results.size()).isEqualTo(1);
} catch (IOException e) {
throw new RuntimeException(e);
}
}
}

157
nbr/build-bin-nbr.sh
View File

@ -1,151 +1,70 @@
#!/usr/bin/env bash
#
# Copyright (c) 2022 nosqlbench
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
set -e
set -x
JAVA_VERSION="16"
APPDIR=target/NBR.AppDir
APPDIR=target/NB.AppDir
JAR_NAME="nbr.jar"
BIN_NAME="nbr"
JAVA_VERSION="17"
mkdir -p ${APPDIR}
BUILD_OPENJ9="false"
if [ "$1" = "--with-openj9" ]
if [ ! -f target/${JAR_NAME} ]
then
BUILD_OPENJ9="true"
printf "using openj9 for build\n"
shift;
fi
if [ ! -f target/nbr.jar ]
then
printf "target/nbr.jar does not exist"
printf "target/${JAR_NAME} does not exist"
exit 2
fi
#if [ -x "target/nbr" ]
#then
# printf "Removing stale target/nbr...\n"
# rm target/nbr
#fi
rsync -av appimage/skel/ "${APPDIR}/"
cp target/nbr.jar "${APPDIR}/usr/bin/nbr.jar"
cp target/${JAR_NAME} "${APPDIR}/usr/bin/${JAR_NAME}"
mkdir -p "${APPDIR}/usr/bin/jre"
if [ "$JAVA_VERSION" == "16" ]
if [ "$JAVA_VERSION" == "17" ]
then
if [ ! -d "cache/jre16" ] ; then
printf "getting jre once into cache/jre15\n";
if [ ! -d "cache/jdk17" ] ; then
printf "getting jdk17 once into cache/jdk17\n";
mkdir -p cache
(cd cache && (
if [ "$BUILD_OPENJ9" = "true" ]
then
printf "BUILD_OPENJ9 is not supported in this version yet\n"
exit 2
else
wget -c 'https://github.com/AdoptOpenJDK/openjdk16-binaries/releases/download/jdk-16.0.1+9/OpenJDK16U-jre_x64_linux_hotspot_16.0.1_9.tar.gz'
tar xf OpenJDK16U-jre_x64_linux_hotspot_16.0.1_9.tar.gz
mv jdk-16.0.1+9-jre jre16
fi
wget -c https://github.com/adoptium/temurin17-binaries/releases/download/jdk-17.0.1%2B12/OpenJDK17U-jdk_x64_linux_hotspot_17.0.1_12.tar.gz
tar -xf OpenJDK17U-jdk_x64_linux_hotspot_17.0.1_12.tar.gz
mv jdk-17.0.1+12 jdk17
rm OpenJDK17U-jdk_x64_linux_hotspot_17.0.1_12.tar.gz
))
fi
rsync -av cache/jre16/ "${APPDIR}/usr/bin/jre/"
elif [ "$JAVA_VERSION" == "15" ]
then
if [ ! -d "cache/jre15" ] ; then
printf "getting jre once into cache/jre15\n";
mkdir -p cache
(cd cache && (
if [ "$BUILD_OPENJ9" = "true" ]
then
wget -c https://github.com/AdoptOpenJDK/openjdk15-binaries/releases/download/jdk15u-2020-11-19-07-04/OpenJDK15U-jre_x64_linux_openj9_linuxXL_2020-11-19-07-04.tar.gz
tar -xf OpenJDK15U-jre_x64_linux_openj9_linuxXL_2020-11-19-07-04.tar.gz
mv jdk-15.0.1+9-jre jre15
rm OpenJDK15U-jre_x64_linux_openj9_linuxXL_2020-11-19-07-04.tar.gz
else
wget -c https://github.com/AdoptOpenJDK/openjdk15-binaries/releases/download/jdk15u-2020-11-19-07-04/OpenJDK15U-jre_x64_linux_hotspot_2020-11-19-07-04.tar.gz
tar xf OpenJDK15U-jre_x64_linux_hotspot_2020-11-19-07-04.tar.gz
mv jdk-15.0.1+9-jre jre15
fi
))
fi
rsync -av cache/jre15/ "${APPDIR}/usr/bin/jre/"
# Java 14 should run binaries targeted to Java 11 bytecode
elif [ "$JAVA_VERSION" == "14" ] ; then
if [ ! -d "cache/jre14" ] ; then
printf "getting jre once into cache/jre14\n";
mkdir -p cache
(cd cache && (
if [ "$BUILD_OPENJ9" = "true" ]
then
wget -c https://github.com/AdoptOpenJDK/openjdk14-binaries/releases/download/jdk14u-2020-04-27-07-27/OpenJDK14U-jre_x64_linux_openj9_linuxXL_2020-04-27-07-27.tar.gz
tar xf OpenJDK14U-jre_x64_linux_openj9_linuxXL_2020-04-27-07-27.tar.gz
mv jdk-14.0.1+7-jre jre14
rm OpenJDK14U-jre_x64_linux_openj9_linuxXL_2020-04-27-07-27.tar.gz
else
wget -c https://github.com/AdoptOpenJDK/openjdk14-binaries/releases/download/jdk14u-2020-04-27-07-27/OpenJDK14U-jre_x64_linux_hotspot_2020-04-27-07-27.tar.gz
tar xf OpenJDK14U-jre_x64_linux_hotspot_2020-04-27-07-27.tar.gz
mv jdk-14.0.1+7-jre jre14
fi
))
fi
rsync -av cache/jre14/ "${APPDIR}/usr/bin/jre/"
rsync -av cache/jdk17/ "${APPDIR}/usr/bin/jre/"
else
printf "Unknown java version indicated in $0"
exit 2
# wget -c https://github.com/AdoptOpenJDK/openjdk12-binaries/releases/download/jdk-12.0.2%2B10/OpenJDK12U-jre_x64_linux_hotspot_12.0.2_10.tar.gz
# tar xf OpenJDK12U-jre_x64_linux_hotspot_12.0.2_10.tar.gz
# mv jdk-12.0.2+10-jre jre
# rm OpenJDK12U-jre_x64_linux_hotspot_12.0.2_10.tar.gz
fi
if [ ! -f "${APPDIR}/AppRun" ]
then
( cd ${APPDIR} && (
printf "Linking AppRun...\n";
ln -s usr/bin/nbr AppRun
ln -s usr/bin/${BIN_NAME} AppRun
))
fi
#( cd ${APPDIR} && (
# rsync -av ..
# if [ ! -d "usr/bin/jre" ]
# then
# printf "getting jre...\n";
#
# # JRE 12
# wget -c https://github.com/AdoptOpenJDK/openjdk12-binaries/releases/download/jdk-12.0.2%2B10/OpenJDK12U-jre_x64_linux_hotspot_12.0.2_10.tar.gz
# tar xf OpenJDK12U-jre_x64_linux_hotspot_12.0.2_10.tar.gz
# mv jdk-12.0.2+10-jre usr/bin/jre
# rm OpenJDK12U-jre_x64_linux_hotspot_12.0.2_10.tar.gz
#
# # JRE 13
# # wget -c https://github.com/AdoptOpenJDK/openjdk13-binaries/releases/download/jdk-13%2B33/OpenJDK13U-jre_x64_linux_hotspot_13_33.tar.gz
# # tar xf OpenJDK13U-jre_x64_linux_hotspot_13_33.tar.gz
# #mv jdk-13+33-jre usr/bin/jre
# #rm OpenJDK13U-jre_x64_linux_hotspot_13_33.tar.gz
# else
# printf "jre directory present, skipping...\n";
# fi
#
# if [ -f "AppRun" ]
# then
# printf "Removing stale AppRun...\n";
# rm AppRun
# fi
#
# if [ ! -f "AppRun" ]
# then
# printf "Linking AppRun...\n";
# ln -s usr/bin/nbr AppRun
# fi
#
# )
#)
printf "getting appimage tool and building image...\n";
( cd target && (
@ -155,13 +74,13 @@ printf "getting appimage tool and building image...\n";
chmod +x appimagetool-x86_64.AppImage
fi
ARCH=x86_64 ./appimagetool-x86_64.AppImage NBR.AppDir nbr
# && chmod +x nbr
ARCH=x86_64 ./appimagetool-x86_64.AppImage NB.AppDir ${BIN_NAME}
# && chmod +x ${BIN_NAME}
)
)
if [ -x "target/nbr" ]
if [ -x "target/${BIN_NAME}" ]
then
printf "nosqlbench AppImage binary was built at target/nbr\n";
printf "nosqlbench AppImage binary was built at target/${BIN_NAME}\n";
fi

3
nbr/pom.xml
View File

@ -150,13 +150,14 @@
</build>
<profiles>
<profile>
<id>build-nb-appimage</id>
<id>build-nbr-appimage</id>
<activation>
<os>
<family>unix</family>
<name>linux</name>
<arch>amd64</arch>
</os>
<activeByDefault>false</activeByDefault>
</activation>
<build>
<plugins>

2
nbr/src/main/java/io/nosqlbench/api/docsapi/docexporter/BundledVersionManifest.java
View File

@ -28,7 +28,7 @@ public class BundledVersionManifest implements BundledMarkdownManifest {
public DocsBinder getDocs() {
return new Docs()
.namespace("versions")
.addPath(NBIO.classpath().name("nb_version_info.md").one().asPath())
.addPath(NBIO.classpath().pathname("nb_version_info.md").one().asPath())
.asDocsBinder();
}
}

21
scripts/trigger-preview-build Executable file
View File

@ -0,0 +1,21 @@
#!/bin/bash
cd mvn-defaults
#set -x
CURRENT_VERSION=$(mvn help:evaluate -Dexpression=revision -q -DforceStdout)
RELEASE_VERSION=${CURRENT_VERSION%%-SNAPSHOT}
PREVIEW_TAG="${RELEASE_VERSION}-preview"
printf "preview tag: '%s'\n" "${PREVIEW_TAG}"
set -e
if [ "$1" == "-f" ]
then
git tag -f "${PREVIEW_TAG}" \
&& git push -f origin "${PREVIEW_TAG}" \
&& printf "force pushed tag %s to origin, triggering build.\n" "${PREVIEW_TAG}"
else
git tag "${PREVIEW_TAG}" \
&& git push origin "${PREVIEW_TAG}" \
&& printf "pushed tag %s to origin, triggering build.\n" "${PREVIEW_TAG}"
fi

21
scripts/trigger-release-build Executable file
View File

@ -0,0 +1,21 @@
#!/bin/bash
cd mvn-defaults
#set -x
CURRENT_VERSION=$(mvn help:evaluate -Dexpression=revision -q -DforceStdout)
RELEASE_VERSION=${CURRENT_VERSION%%-SNAPSHOT}
RELEASE_TAG="${RELEASE_VERSION}-release"
printf "release tag: '%s'\n" "${RELEASE_TAG}"
set -e
if [ "$1" == "-f" ]
then
git tag -f "${RELEASE_TAG}" \
&& git push -f origin "${RELEASE_TAG}" \
&& printf "force pushed tag %s to origin, triggering build.\n" "${RELEASE_TAG}"
else
git tag "${RELEASE_TAG}" \
&& git push origin "${RELEASE_TAG}" \
&& printf "pushed tag %s to origin, triggering build.\n" "${RELEASE_TAG}"
fi

2
scripts/trigger_preview_build.sh
View File

@ -1,2 +0,0 @@
#!/bin/bash
scripts/release-notes.sh > PREVIEW_NOTES.md

2
scripts/trigger_release_build.sh
View File

@ -1,2 +0,0 @@
#!/bin/bash
scripts/release-notes.sh > RELEASE_NOTES.md

3
virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/core/threadstate/SharedState.java
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 2022 nosqlbench
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -21,6 +21,7 @@ import java.util.concurrent.ConcurrentHashMap;
/**
* This provides common thread local instancing for sharing a thread local map across classes.
* This is being described as a <em>Thread Local State Cache</em>.
*/
public class SharedState {

39
virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_object/Discard.java Normal file
View File

@ -0,0 +1,39 @@
/*
* Copyright (c) 2022-2023 nosqlbench
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.nosqlbench.virtdata.library.basics.shared.from_long.to_object;
import io.nosqlbench.virtdata.api.annotations.Categories;
import io.nosqlbench.virtdata.api.annotations.Category;
import io.nosqlbench.virtdata.api.annotations.ThreadSafeMapper;
import java.util.function.LongFunction;
/**
* This function takes a long input and ignores it. It returns a generic object which is meant to be used as input to
* other function which don't need a specific input.
*/
@ThreadSafeMapper
@Categories({Category.general})
public class Discard implements LongFunction<Object> {
Object object = "discard";
@Override
public Object apply(long value) {
return object;
}
}