IntenseFinance/nosqlbench
3
0
Fork 0
mirror of https://github.com/nosqlbench/nosqlbench.git synced 2025-02-25 18:55:28 -06:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'listfuncs'

Browse Source 
# Conflicts:
#	activitytype-stdout/src/main/java/io/nosqlbench/activitytype/stdout/StdoutActivity.java
#	docsys/src/main/resources/docs-for-docsys/docsys/design/topics.md
#	virtdata-api/src/main/java/io/nosqlbench/virtdata/core/bindings/VirtDataComposer.java
#	virtdata-api/src/main/java/io/nosqlbench/virtdata/core/templates/StringBindingsTemplate.java
#	virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_collection/ListHashed.java
#	virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_collection/ListSized.java
#	virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_collection/ListSizedHashed.java
#	virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_collection/ListSizedStepped.java
#	virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_collection/ListStepped.java
This commit is contained in:
Jonathan Shook 2020-04-23 16:21:28 -05:00
commit 680ff3b3f8
16 changed files with 233 additions and 207 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
activitytype-stdout/src/main/java/io/nosqlbench/activitytype/stdout/StdoutActivity.java
View File

@ -138,13 +138,23 @@ public class StdoutActivity extends SimpleActivity implements ActivityDefObserve
String format = getParams().getOptionalString("format").orElse(null);
if ((stmts.size()==0 && stmtsDocList.getDocBindings().size() > 0) || format!=null) {
logger.info("Creating stdout statement template from bindings...");
String generatedStmt = genStatementTemplate(stmtsDocList.getDocBindings().keySet());
BindingsTemplate bt = new BindingsTemplate();
stmtsDocList.getDocBindings().forEach(bt::addFieldBinding);
StringBindingsTemplate sbt = new StringBindingsTemplate(generatedStmt, bt);
StringBindings sb = sbt.resolve();
sequencer.addOp(sb,1L);
if (format!=null && format.startsWith("diag")) {
logger.info("Creating diagnostic log for resolver construction...");
BindingsTemplate bt = new BindingsTemplate();
stmtsDocList.getDocBindings().forEach(bt::addFieldBinding);
String diagnostics = bt.getDiagnostics();
System.out.println(diagnostics);
System.out.flush();
System.exit(2);
} else {
logger.info("Creating stdout statement template from bindings, since none is otherwise defined.");
String generatedStmt = genStatementTemplate(stmtsDocList.getDocBindings().keySet());
BindingsTemplate bt = new BindingsTemplate();
stmtsDocList.getDocBindings().forEach(bt::addFieldBinding);
StringBindingsTemplate sbt = new StringBindingsTemplate(generatedStmt, bt);
StringBindings sb = sbt.resolve();
sequencer.addOp(sb,1L);
}
} else if (stmts.size() > 0) {
for (StmtDef stmt : stmts) {
ParsedStmt parsed = stmt.getParsed().orError();

8
activitytype-stdout/src/main/resources/stdout.md
View File

@ -33,9 +33,11 @@ activity types.
- **newline** - whether to automatically add a missing newline to the end
of any statements.
default: true
- **format** - which format to use. If provided, the format will override
any statement formats provided by the YAML.
valid values are (csv, readout, json, inlinejson, and assignments)
- **format** - which format to use. If provided, the format will override any statement formats provided by the YAML.
valid values are (csv, readout, json, inlinejson, assignments, and diag)
- When 'format=diag', then the internal construction logic for the binding is logged in detail and nosqlbench exits.
This is useful for detailed diagnostics when you run into trouble, but not generally otherwise. This provides
details that you may include in a bug report if you think there is a bindings bug.
## Configuration

0
docsys/src/main/resources/docs-for-docsys/docsys/design/dynamic_endpoints.md → devdocs/dynamic_endpoints.md
View File

15
devdocs/inspirations.md Normal file
View File

@ -0,0 +1,15 @@
# Virtual DocSys
## Inspiring Examples
These are doc sites that have examples of good docs.
- [Apache Groovy](http://groovy-lang.org/documentation.html)
- [Prometheus](https://prometheus.io/docs/prometheus/latest/querying/basics/)
- [NetData](https://docs.netdata.cloud/)
- [Optimizely](https://developers.optimizely.com/x/solutions/javascript/reference/index.html)
- [Elastic Step by Step](https://www.elastic.co/guide/en/elasticsearch/client/java-rest/current/java-rest-high-search.html)
- [Javascript.info](https://javascript.info/strict-mode)
- [TiKV docs - explore the tabs](https://tikv.org/docs/3.0/concepts/overview/)
- [rocket](https://rocket.rs/v0.4/overview/)

77
docsys/src/main/resources/docs-for-docsys/docsys/design/docs_intro.md
View File

@ -1,77 +0,0 @@
# Virtual DocSys
## Inspiring Examples
These are doc sites that have examples of good docs.
- [Apache Groovy](http://groovy-lang.org/documentation.html)
- [Prometheus](https://prometheus.io/docs/prometheus/latest/querying/basics/)
- [NetData](https://docs.netdata.cloud/)
- [Optimizely](https://developers.optimizely.com/x/solutions/javascript/reference/index.html)
- [Elastic Step by Step](https://www.elastic.co/guide/en/elasticsearch/client/java-rest/current/java-rest-high-search.html)
- [Javascript.info](https://javascript.info/strict-mode)
- [TiKV docs - explore the tabs](https://tikv.org/docs/3.0/concepts/overview/)
- [rocket](https://rocket.rs/v0.4/overview/)
## Architecture
The RealSimpleDoc system is meant to be embedded within
other projects. It has been design to require a minimum of
additional programming at the web layer, while allowing the tool
builder to focus on direct content transforms of file types.
It is also designed to allow multiple projects to contribute
documentation from their constituent components, with
the contents being layered and composited dynamically.
Thus, the primary interface to the web server is provided as a filesystem
instance that presents the directory structure and file content as
the designer would want the user to see it, including content from
multiple contributing components. This shifts the classic problem
of doing server-side web programming in-depth for basic content
authoring to one of simply having the right transformers in place.
## MetaFS
The content provided to the web server from the filesystem is provided
by a set of filesystem modules that are collectively called MetaFS. This
consists of three specific filesystems which each serve a simple purpose:
### VirtualFS
The VirtualFS filesystem type is simply a way to provide a view to a
filesystem that is rooted at some directory path in a host filesystem.
For example, a VirtFS that is created with a host filesystem path of
`/usr/local/branson` will contain a Path entry for `/` which will look
and behave like a root directory, but all contents accessed via this
path will come directly from the host's filesystem `/usr/local/branson`.
### LayerFS
This filesystem type implements the ability to combine multiple filesystems
in a layered fashion. Any attempt to access a file or directory in this
filesystem will cause an internal request to the filesystems that have
been added. LayerFS follows a couple basic rules when answering a request:
1. For calls that access attributes, the first response that
contains attributes for a file that does exist will be used.
2. When opening a file for write, the first filesystem which
is writable will be used to open the file.
3. Any requests which ask whether a file is readable or writable
will have their answers filtered to match the effective best case.
### RenderFS
The RenderFS filesystem takes a set of transformers that are associated with a
source and target file extension. The rules observed by this filesystem are:
1. For directory listings, if there is a file with one of the source extensions,
but the target file for that basename does not exist, then the directory listing
is modified to show both.
2. For file attribute views, if a call fails internally to find file attributes,
and a source version of the base name exist for that extension, then a virtual
attribute view is created that has all the same attributes as the base name, but
with a different file name.
3. If there is attempt to read file contents which are of a known target extension,
where the source file exists, the content will be rendered from the source file
and returned.

16
engine-api/src/test/java/io/nosqlbench/engine/api/activityconfig/rawyaml/BindingEscapingTest.java
View File

@ -30,10 +30,22 @@ public class BindingEscapingTest {
private final static Logger logger = LoggerFactory.getLogger(BindingEscapingTest.class);
@Test
public void testEscapedBindings() {
DataMapper<String> mapper = VirtData.getMapper("Template('\"-{}-\"Func(234)\\\"\\)',NumberNameToString());'",String.class);
public void testThatEscapesAreNotUnescapedInSingleQuotes() {
DataMapper<String> mapper = VirtData.getMapper(
"Template('\"-{}-\"Func(234)\\\"\\)',NumberNameToString());"
);
String s = mapper.get(234);
assertThat(s).isEqualTo("\"-two hundred and thirty four-\"Func(234)\\\"\\)");
}
@Test
public void testThatEscapesAreUnescapedInSingleQuotes() {
DataMapper<String> mapper =
VirtData.getMapper(
"Template(\"\\\"-{}-\\\"Func(234)\\\")\",NumberNameToString());"
);
String s = mapper.get(234);
assertThat(s).isEqualTo("\"-two hundred and thirty four-\"Func(234)\")");
}
}

41
virtdata-api/src/main/java/io/nosqlbench/virtdata/core/bindings/BindingsTemplate.java
View File

@ -38,7 +38,7 @@ import java.util.Optional;
* bindings will be used in.
*/
public class BindingsTemplate {
private final static Logger logger = LogManager.getLogger(BindingsTemplate.class);
private final static Logger logger = LogManager.getLogger(BindingsTemplate.class);
private List<String> bindPointNames = new ArrayList<>();
private List<String> specifiers = new ArrayList<>();
@ -47,11 +47,11 @@ public class BindingsTemplate {
// }
public BindingsTemplate(List<String> anchors, List<String> specs) {
if (anchors.size()!=specs.size()) {
if (anchors.size() != specs.size()) {
throw new InvalidParameterException("Anchors and Specifiers must be matched pair-wise.");
}
for (int i = 0; i < anchors.size(); i++) {
addFieldBinding(anchors.get(i),specs.get(i));
addFieldBinding(anchors.get(i), specs.get(i));
}
}
@ -64,14 +64,15 @@ public class BindingsTemplate {
public void addFieldBindings(List<BindPoint> bindPoints) {
for (BindPoint bindPoint : bindPoints) {
addFieldBinding(bindPoint.getAnchor(),bindPoint.getBindspec());
addFieldBinding(bindPoint.getAnchor(), bindPoint.getBindspec());
}
}
/**
* Add a named binding specifier to the template
*
* @param bindPointName the name associated with the binding specifier
* @param genSpec the binding specifier
* @param genSpec the binding specifier
*/
public void addFieldBinding(String bindPointName, String genSpec) {
this.bindPointNames.add(bindPointName);
@ -80,19 +81,39 @@ public class BindingsTemplate {
/**
* Add multiple named bindings to the template
*
* @param bindPairs A map of named binding specifiers
*/
public void addFieldBindings(Map<String,String> bindPairs) {
public void addFieldBindings(Map<String, String> bindPairs) {
for (Map.Entry<String, String> e : bindPairs.entrySet()) {
this.bindPointNames.add(e.getKey());
this.specifiers.add(e.getValue());
}
}
public String getDiagnostics() {
StringBuilder diaglog = new StringBuilder();
for (String specifier : specifiers) {
diaglog.append("for ").append(specifier).append(":");
ResolverDiagnostics mapperDiagnostics = VirtData.getMapperDiagnostics(specifier);
String diagnostics = mapperDiagnostics.toString();
diaglog.append(diagnostics);
if (mapperDiagnostics.getResolvedFunction().isPresent()) {
diaglog.append("☑ RESOLVED:")
.append(mapperDiagnostics.getResolvedFunction().get().toString()).append("\n");
} else {
diaglog.append("☐ UNRESOLVED\n");
}
}
return diaglog.toString();
}
/**
* Use the data mapping library and the specifier to create instances of data mapping functions.
* If you need thread-aware mapping, be sure to call this in the proper thread. Each time this method
* is called, it creates a new instance.
*
* @return A set of bindings that can be used to yield mapped data values later.
*/
public Bindings resolveBindings() {
@ -104,9 +125,9 @@ public class BindingsTemplate {
} else {
logAvailableDataMappers();
throw new RuntimeException(
"data mapper binding was unsuccessful for "
+ ", spec:" + specifier
+ ", see log for known data mapper names.");
"data mapper binding was unsuccessful for "
+ ", spec:" + specifier
+ ", see log for known data mapper names.");
}
}
return new Bindings(this, dataMappers);
@ -149,7 +170,7 @@ public class BindingsTemplate {
sb.append("=>[");
sb.append(values[i]);
sb.append("](");
sb.append((null!=values[i]) ? values[i].getClass().getSimpleName() : "NULL");
sb.append((null != values[i]) ? values[i].getClass().getSimpleName() : "NULL");
sb.append(")");
delim = ", ";
}

64
virtdata-api/src/main/java/io/nosqlbench/virtdata/core/bindings/VirtDataComposer.java
View File

@ -50,12 +50,12 @@ import java.util.stream.Collectors;
public class VirtDataComposer {
private final static String PREAMBLE = "compose ";
private final static Logger logger = LogManager.getLogger(DataMapperLibrary.class);
private final static Logger logger = LogManager.getLogger(DataMapperLibrary.class);
private final static MethodHandles.Lookup lookup = MethodHandles.publicLookup();
private final VirtDataFunctionLibrary functionLibrary;
private final Map<String,Object> customElements = new HashMap<>();
private final Map<String, Object> customElements = new HashMap<>();
public VirtDataComposer(VirtDataFunctionLibrary functionLibrary) {
this.functionLibrary = functionLibrary;
@ -91,23 +91,22 @@ public class VirtDataComposer {
public ResolverDiagnostics resolveDiagnosticFunctionFlow(VirtDataFlow flow) {
ResolverDiagnostics diagnostics = new ResolverDiagnostics();
diagnostics.trace("processing flow " + flow.toString() + " from output to input");
LinkedList<List<ResolvedFunction>> funcs = new LinkedList<>();
LinkedList<Set<Class<?>>> nextFunctionInputTypes = new LinkedList<>();
Optional<Class<?>> finalValueTypeOption =
Optional.ofNullable(flow.getLastExpression().getCall().getOutputType())
.map(ValueType::valueOfClassName).map(ValueType::getValueClass);
Optional.ofNullable(flow.getLastExpression().getCall().getOutputType())
.map(ValueType::valueOfClassName).map(ValueType::getValueClass);
nextFunctionInputTypes.add(new HashSet<>());
finalValueTypeOption.ifPresent(t -> nextFunctionInputTypes.get(0).add(t));
diagnostics.trace("working backwards from " + (flow.getExpressions().size()-1));
diagnostics.trace("working backwards from index " + (flow.getExpressions().size() - 1) + " to index 0");
for (int i = flow.getExpressions().size() - 1; i >= 0; i--) {
FunctionCall call = flow.getExpressions().get(i).getCall();
diagnostics.trace("resolving args for " + call.toString());
diagnostics.trace("FUNCTION[" + i + "]: " + call.toString() + ", resolving args");
// diagnostics.trace("resolving args for " + call.toString());
List<ResolvedFunction> nodeFunctions = new LinkedList<>();
@ -115,23 +114,24 @@ public class VirtDataComposer {
Class<?> inputType = ValueType.classOfType(call.getInputType());
Class<?> outputType = ValueType.classOfType(call.getOutputType());
Object[] args = call.getArguments();
try {
args = populateFunctions(diagnostics, args, this.customElements);
} catch (Exception e) {
return diagnostics.error(e);
}
diagnostics.trace("resolved args: ");
diagnostics.trace(" resolved args:");
for (Object arg : args) {
diagnostics.trace(" " + arg.getClass().getSimpleName() + ": " + arg.toString());
diagnostics.trace(" " + arg.getClass().getSimpleName() + ": " + arg.getClass().getCanonicalName());
}
List<ResolvedFunction> resolved = functionLibrary.resolveFunctions(outputType, inputType, funcName, this.customElements,args);
List<ResolvedFunction> resolved = functionLibrary.resolveFunctions(outputType, inputType, funcName, this.customElements, args);
if (resolved.size() == 0) {
return diagnostics.error(new RuntimeException("Unable to find even one function for " + call));
}
diagnostics.trace(" resolved functions:");
diagnostics.trace(summarize(resolved));
diagnostics.trace(" resolved functions");
diagnostics.trace(summarize(resolved, " - "));
nodeFunctions.addAll(resolved);
funcs.addFirst(nodeFunctions);
@ -153,10 +153,10 @@ public class VirtDataComposer {
}
FunctionAssembly assembly = new FunctionAssembly();
diagnostics.trace("composed summary: " + summarize(flattenedFuncs));
// diagnostics.trace("composed summary: " + summarize(flattenedFuncs));
boolean isThreadSafe = true;
diagnostics.trace("FUNCTION chain selected: (multi) '" + this.summarize(flattenedFuncs) + "'");
diagnostics.trace("FUNCTION chain selected: (multi) '" + this.summarize(flattenedFuncs, " - ") + "'");
for (ResolvedFunction resolvedFunction : flattenedFuncs) {
try {
Object functionObject = resolvedFunction.getFunctionObject();
@ -165,7 +165,7 @@ public class VirtDataComposer {
isThreadSafe = false;
}
} catch (Exception e) {
String flowdata = flow!=null? flow.toString() : "undefined";
String flowdata = flow != null ? flow.toString() : "undefined";
return diagnostics.error(new RuntimeException("FUNCTION resolution failed: '" + flowdata + "': " + e.toString()));
}
}
@ -179,7 +179,7 @@ public class VirtDataComposer {
return resolverDiagnostics.getResolvedFunction();
}
private Object[] populateFunctions(ResolverDiagnostics diagnostics, Object[] args, Map<String,?> cconfig) {
private Object[] populateFunctions(ResolverDiagnostics diagnostics, Object[] args, Map<String, ?> cconfig) {
for (int i = 0; i < args.length; i++) {
Object o = args[i];
if (o instanceof FunctionCall) {
@ -189,7 +189,8 @@ public class VirtDataComposer {
Class<?> inputType = ValueType.classOfType(call.getInputType());
Class<?> outputType = ValueType.classOfType(call.getOutputType());
Object[] fargs = call.getArguments();
diagnostics.trace("resolving argument as function '" + call.toString() + "'");
diagnostics.trace(" arg (function): " + call.toString());
// diagnostics.trace("resolving argument as function '" + call.toString() + "'");
fargs = populateFunctions(diagnostics, fargs, cconfig);
List<ResolvedFunction> resolved = functionLibrary.resolveFunctions(outputType, inputType, funcName, cconfig, fargs);
@ -216,9 +217,11 @@ public class VirtDataComposer {
funcs.removeAll(toRemove);
}
private String summarize(List<ResolvedFunction> funcs) {
private String summarize(List<ResolvedFunction> funcs, String prefix) {
return funcs.stream()
.map(String::valueOf).collect(Collectors.joining("|"));
.map(String::valueOf)
.map(s -> prefix + s)
.collect(Collectors.joining("\n"));
}
private String summarizeBulk(List<List<ResolvedFunction>> funcs) {
@ -226,9 +229,9 @@ public class VirtDataComposer {
List<List<String>> spans = new LinkedList<>();
funcs.forEach(l -> spans.add(l.stream().map(String::valueOf).collect(Collectors.toList())));
List<Optional<Integer>> widths = spans.stream().map(
l -> l.stream().map(String::length).max(Integer::compare)).collect(Collectors.toList());
l -> l.stream().map(String::length).max(Integer::compare)).collect(Collectors.toList());
String funcsdata = spans.stream().map(
l -> l.stream().map(String::valueOf).collect(Collectors.joining("|\n"))
l -> l.stream().map(String::valueOf).collect(Collectors.joining("|\n"))
).collect(Collectors.joining("\n\n"));
StringBuilder sb = new StringBuilder();
@ -322,7 +325,7 @@ public class VirtDataComposer {
funcs.sort(ResolvedFunction.PREFERRED_TYPE_COMPARATOR);
while (funcs.size() > 1) {
logger.trace("BY-SINGLE-PREFERRED-TYPE removing func " + funcs.get(funcs.size() - 1)
+ " because " + funcs.get(0) + " has more preferred types.");
+ " because " + funcs.get(0) + " has more preferred types.");
funcs.remove(funcs.size() - 1);
}
@ -337,7 +340,7 @@ public class VirtDataComposer {
prevFuncs.sort(ResolvedFunction.PREFERRED_TYPE_COMPARATOR);
while (prevFuncs.size() > 1) {
String logmsg = "BY-PREV-PREFERRED-TYPE removing func " + prevFuncs.get(prevFuncs.size() - 1)
+ " because " + prevFuncs.get(0) + " has more preferred types.";
+ " because " + prevFuncs.get(0) + " has more preferred types.";
logger.trace(logmsg);
prevFuncs.remove(prevFuncs.size() - 1);
}
@ -346,7 +349,7 @@ public class VirtDataComposer {
nextFuncs.sort(ResolvedFunction.PREFERRED_TYPE_COMPARATOR);
while (nextFuncs.size() > 1) {
String logmsg = "BY-NEXT-PREFERRED-TYPE removing func " + nextFuncs.get(nextFuncs.size() - 1)
+ " because " + nextFuncs.get(0) + " has more preferred types.";
+ " because " + nextFuncs.get(0) + " has more preferred types.";
logger.trace(logmsg);
nextFuncs.remove(nextFuncs.size() - 1);
}
@ -370,7 +373,7 @@ public class VirtDataComposer {
Set<Class<?>> outputs = getOutputs(prevFuncs);
Set<Class<?>> inputs = getInputs(nextFuncs);
Set<Class<?>> directMatches =
inputs.stream().filter(outputs::contains).collect(Collectors.toCollection(HashSet::new));
inputs.stream().filter(outputs::contains).collect(Collectors.toCollection(HashSet::new));
if (directMatches.size() > 0) {
List<ResolvedFunction> toremove = new ArrayList<>();
@ -424,9 +427,9 @@ public class VirtDataComposer {
return 0;
} else {
toremove.forEach(nextfunc -> {
String logmsg = "BY-ASSIGNABLE-TYPE removing next func: " + nextfunc + " because its input types are not assignable from any of the previous funcs";
logger.trace(logmsg);
}
String logmsg = "BY-ASSIGNABLE-TYPE removing next func: " + nextfunc + " because its input types are not assignable from any of the previous funcs";
logger.trace(logmsg);
}
);
nextFuncs.removeAll(toremove);
@ -451,9 +454,10 @@ public class VirtDataComposer {
return inputs;
}
public Map<String,?> getCustomElements() {
public Map<String, ?> getCustomElements() {
return this.customElements;
}
public VirtDataComposer addCustomElement(String name, Object element) {
this.customElements.put(name, element);
return this;

10
virtdata-api/src/main/java/io/nosqlbench/virtdata/core/templates/StringBindingsTemplate.java
View File

@ -63,6 +63,16 @@ public class StringBindingsTemplate {
return new StringBindings(compositor,bindings);
}
public String getDiagnostics() {
StringCompositor compositor = new StringCompositor(stringTemplate);
HashSet<String> unqualifiedNames = new HashSet<>(compositor.getBindPointNames());
unqualifiedNames.removeAll(new HashSet<>(bindingsTemplate.getBindPointNames()));
if (unqualifiedNames.size()>0) {
throw new RuntimeException("Named anchors were specified in the template which were not provided in the bindings: " + unqualifiedNames.toString());
}
return bindingsTemplate.getDiagnostics();
}
@Override
public String toString() {
return "TEMPLATE:"+this.stringTemplate+" BINDING:"+bindingsTemplate.toString();

11
virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_collection/ListHashed.java
View File

@ -30,15 +30,13 @@ public class ListHashed implements LongFunction<List<Object>> {
private final List<LongFunction<? extends Object>> valueFuncs;
private final int size;
private final LongToIntFunction sizeFunc;
private final Hash hasher = new Hash();
@Example({
"ListFunctions(NumberNameToString(),NumberNameToString())",
"Create a list of ['one','one']"
})
public ListHashed(LongToIntFunction sizeFunc, LongFunction<? extends Object>... funcs) {
this.sizeFunc = sizeFunc;
public ListHashed(LongFunction<? extends Object>... funcs) {
this.valueFuncs = Arrays.asList(funcs);
this.size = valueFuncs.size();
}
@ -47,12 +45,11 @@ public class ListHashed implements LongFunction<List<Object>> {
"ListFunctions(NumberNameToString(),NumberNameToString())",
"Create a list of ['one','one']"
})
public ListHashed(LongToIntFunction sizeFunc, LongUnaryOperator... funcs) {
public ListHashed(LongUnaryOperator... funcs) {
List<LongFunction<?>> building = new ArrayList<>(funcs.length);
for (LongUnaryOperator func : funcs) {
building.add(func::applyAsLong);
}
this.sizeFunc = sizeFunc;
this.valueFuncs = building;
this.size = building.size();
}
@ -61,12 +58,11 @@ public class ListHashed implements LongFunction<List<Object>> {
"ListFunctions(NumberNameToString(),NumberNameToString())",
"Create a list of ['one','one']"
})
public ListHashed(LongToIntFunction sizeFunc, Function<Long,Object>... funcs) {
public ListHashed(Function<Long,Object>... funcs) {
List<LongFunction<?>> building = new ArrayList<>(funcs.length);
for (Function<Long,Object> func : funcs) {
building.add(func::apply);
}
this.sizeFunc = sizeFunc;
this.valueFuncs = building;
this.size = building.size();
}
@ -74,7 +70,6 @@ public class ListHashed implements LongFunction<List<Object>> {
@Override
public List<Object> apply(long value) {
long hash = value;
int size = sizeFunc.applyAsInt(value);
List<Object> list = new ArrayList<>(size);
for (int i = 0; i < size; i++) {
int selector = Math.min(i, valueFuncs.size() - 1);

4
virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_collection/ListSized.java
View File

@ -28,7 +28,6 @@ import java.util.function.LongUnaryOperator;
public class ListSized implements LongFunction<List<Object>> {
private final List<LongFunction<? extends Object>> valueFuncs;
private final int size;
private final LongToIntFunction sizeFunc;
@Example({
@ -38,7 +37,6 @@ public class ListSized implements LongFunction<List<Object>> {
public ListSized(LongToIntFunction sizeFunc, LongFunction<? extends Object>... funcs) {
this.sizeFunc = sizeFunc;
this.valueFuncs = Arrays.asList(funcs);
this.size = valueFuncs.size();
}
@Example({
@ -52,7 +50,6 @@ public class ListSized implements LongFunction<List<Object>> {
}
this.sizeFunc = sizeFunc;
this.valueFuncs = building;
this.size = building.size();
}
@Example({
@ -66,7 +63,6 @@ public class ListSized implements LongFunction<List<Object>> {
}
this.sizeFunc = sizeFunc;
this.valueFuncs = building;
this.size = building.size();
}
@Override

4
virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_collection/ListSizedHashed.java
View File

@ -29,7 +29,6 @@ import java.util.function.LongUnaryOperator;
public class ListSizedHashed implements LongFunction<List<Object>> {
private final List<LongFunction<? extends Object>> valueFuncs;
private final int size;
private final Hash hasher = new Hash();
private final LongToIntFunction sizeFunc;
@ -40,7 +39,6 @@ public class ListSizedHashed implements LongFunction<List<Object>> {
public ListSizedHashed(LongToIntFunction sizeFunc, LongFunction<? extends Object>... funcs) {
this.sizeFunc = sizeFunc;
this.valueFuncs = Arrays.asList(funcs);
this.size = valueFuncs.size();
}
@Example({
@ -54,7 +52,6 @@ public class ListSizedHashed implements LongFunction<List<Object>> {
}
this.sizeFunc = sizeFunc;
this.valueFuncs = building;
this.size = building.size();
}
@Example({
@ -68,7 +65,6 @@ public class ListSizedHashed implements LongFunction<List<Object>> {
}
this.sizeFunc = sizeFunc;
this.valueFuncs = building;
this.size = building.size();
}
@Override

4
virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_collection/ListSizedStepped.java
View File

@ -29,7 +29,6 @@ import java.util.function.LongUnaryOperator;
public class ListSizedStepped implements LongFunction<List<Object>> {
private final List<LongFunction<? extends Object>> valueFuncs;
private final int size;
private final LongToIntFunction sizeFunc;
@Example({
@ -39,7 +38,6 @@ public class ListSizedStepped implements LongFunction<List<Object>> {
public ListSizedStepped(LongToIntFunction sizeFunc, LongFunction<? extends Object>... funcs) {
this.sizeFunc = sizeFunc;
this.valueFuncs = Arrays.asList(funcs);
this.size = valueFuncs.size();
}
@Example({
@ -53,7 +51,6 @@ public class ListSizedStepped implements LongFunction<List<Object>> {
}
this.sizeFunc = sizeFunc;
this.valueFuncs = building;
this.size = building.size();
}
@Example({
@ -67,7 +64,6 @@ public class ListSizedStepped implements LongFunction<List<Object>> {
}
this.sizeFunc = sizeFunc;
this.valueFuncs = building;
this.size = building.size();
}
@Override

11
virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_collection/ListStepped.java
View File

@ -29,14 +29,12 @@ public class ListStepped implements LongFunction<List<Object>> {
private final List<LongFunction<? extends Object>> valueFuncs;
private final int size;
private final LongToIntFunction sizeFunc;
@Example({
"ListFunctions(NumberNameToString(),NumberNameToString())",
"Create a list of ['one','one']"
})
public ListStepped(LongToIntFunction sizeFunc, LongFunction<? extends Object>... funcs) {
this.sizeFunc = sizeFunc;
public ListStepped(LongFunction<? extends Object>... funcs) {
this.valueFuncs = Arrays.asList(funcs);
this.size = valueFuncs.size();
}
@ -45,12 +43,11 @@ public class ListStepped implements LongFunction<List<Object>> {
"ListFunctions(NumberNameToString(),NumberNameToString())",
"Create a list of ['one','one']"
})
public ListStepped(LongToIntFunction sizeFunc, LongUnaryOperator... funcs) {
public ListStepped(LongUnaryOperator... funcs) {
List<LongFunction<?>> building = new ArrayList<>(funcs.length);
for (LongUnaryOperator func : funcs) {
building.add(func::applyAsLong);
}
this.sizeFunc = sizeFunc;
this.valueFuncs = building;
this.size = building.size();
}
@ -59,19 +56,17 @@ public class ListStepped implements LongFunction<List<Object>> {
"ListFunctions(NumberNameToString(),NumberNameToString())",
"Create a list of ['one','one']"
})
public ListStepped(LongToIntFunction sizeFunc, Function<Long,Object>... funcs) {
public ListStepped(Function<Long,Object>... funcs) {
List<LongFunction<?>> building = new ArrayList<>(funcs.length);
for (Function<Long,Object> func : funcs) {
building.add(func::apply);
}
this.sizeFunc = sizeFunc;
this.valueFuncs = building;
this.size = building.size();
}
@Override
public List<Object> apply(long value) {
int size = sizeFunc.applyAsInt(value);
List<Object> list = new ArrayList<>(size);
for (int i = 0; i < size; i++) {
int selector = Math.min(i, valueFuncs.size() - 1);

128
virtdata-lib-basics/src/main/java/io/nosqlbench/virtdata/library/basics/shared/from_long/to_string/Template.java
View File

@ -19,6 +19,9 @@ import java.util.regex.Pattern;
* result of the provided functions. The number of <code>{}</code> entries in the template
* must strictly match the number of functions or an error will be thrown.
*
* If you need to include single quotes or other special characters, you may use a
* backslash "\" in your template.
*
* The objects passed must be functions of any of the following types:
* <UL>
* <LI>LongUnaryOperator</LI>
@ -66,7 +69,82 @@ public class Template implements LongFunction<String> {
return adapted.toArray(new LongFunction<?>[0]);
}
// public Template(String template, LongFunction<?>... funcs) {
/**
* If an operator is provided, it is used to change the function input value in an additional way before each function.
*
* @param iterOp A pre-generation value mapping function
* @param template A string template containing <pre>{}</pre> anchors
* @param funcs A varargs length of LongFunctions of any output type
*/
public Template(LongUnaryOperator iterOp, String template, LongFunction<?>... funcs) {
this(template, funcs);
this.iterOp = iterOp;
}
@SuppressWarnings("unchecked")
private String[] parseTemplate(String template, int funcCount) {
try {
List<String> literals = new ArrayList<>();
Pattern p = Pattern.compile("\\{}");
Matcher m = p.matcher(template);
int pos = 0;
while (m.find()) {
literals.add(template.substring(pos, m.start()));
pos = m.end();
}
String partial = template.substring(pos);
// partial = unescape(partial);
literals.add(partial);
if (literals.size() != funcCount + 1) {
throw new RuntimeException("The number of {} place holders in '" + template + "' should equal the number of functions.");
}
return literals.toArray(new String[0]);
} catch (Exception e) {
throw new RuntimeException(e);
}
}
// // for testing
// public String unescape(String partial) {
// StringBuilder unescaped = new StringBuilder();
// try {
// Pattern escapes = Pattern.compile("\\\\.");
// Matcher m = escapes.matcher(partial);
// int pos = 0;
//
// while (m.find()) {
// String prefix = partial.substring(pos,m.start());
// unescaped.append(prefix);
// String segment = partial.substring(m.start(),m.end());
// unescaped.append(segment.substring(1));
// pos = m.end();
// }
// unescaped.append(partial.substring(pos));
//
// } catch (Exception e) {
// throw new RuntimeException(e);
// }
// return unescaped.toString();
// }
@Override
public String apply(long value) {
StringBuilder buffer = sb.get();
buffer.setLength(0);
buffer.append(literals[0]);
if (literals.length > 1) {
for (int i = 0; i < adaptedFuncs.length; i++) {
long input = iterOp != null ? iterOp.applyAsLong(value + i) : value + i;
String genString = String.valueOf(adaptedFuncs[i].apply(input));
buffer.append(genString);
buffer.append(literals[i + 1]);
}
}
return buffer.toString();
}
// public Template(String template, LongFunction<?>... funcs) {
// this.adaptedFuncs = funcs;
// this.rawTemplate = template;
// this.literals = parseTemplate(template, funcs.length);
@ -134,52 +212,4 @@ public class Template implements LongFunction<String> {
//
// }
/**
* If an operator is provided, it is used to change the function input value in an additional way before each function.
*
* @param iterOp A pre-generation value mapping function
* @param template A string template containing <pre>{}</pre> anchors
* @param funcs A varargs length of LongFunctions of any output type
*/
public Template(LongUnaryOperator iterOp, String template, LongFunction<?>... funcs) {
this(template, funcs);
this.iterOp = iterOp;
}
@SuppressWarnings("unchecked")
private String[] parseTemplate(String template, int funcCount) {
try {
List<String> literals = new ArrayList<>();
Pattern p = Pattern.compile("\\{}");
Matcher m = p.matcher(template);
int pos = 0;
while (m.find()) {
literals.add(template.substring(pos, m.start()));
pos = m.end();
}
literals.add(template.substring(pos));
if (literals.size() != funcCount + 1) {
throw new RuntimeException("The number of {} place holders in '" + template + "' should equal the number of functions.");
}
return literals.toArray(new String[0]);
} catch (Exception e) {
throw new RuntimeException(e);
}
}
@Override
public String apply(long value) {
StringBuilder buffer = sb.get();
buffer.setLength(0);
buffer.append(literals[0]);
if (literals.length > 1) {
for (int i = 0; i < adaptedFuncs.length; i++) {
long input = iterOp != null ? iterOp.applyAsLong(value + i) : value + i;
String genString = String.valueOf(adaptedFuncs[i].apply(input));
buffer.append(genString);
buffer.append(literals[i + 1]);
}
}
return buffer.toString();
}
}

23
virtdata-lib-basics/src/test/java/io/nosqlbench/virtdata/library/basics/shared/unary_string/TemplateTest.java
View File

@ -52,4 +52,25 @@ public class TemplateTest {
}
}
}
// @Test
// public void testUnescaping() {
// String escaped="front \\' back";
// LongFunction<String> func = String::valueOf;
// Template template = new Template("{} extra", func);
// String unescaped = template.unescape(escaped);
// assertThat(unescaped).isEqualTo("front ' back");
//
// String unescaped2= template.unescape("\\' one \\' two \\'");
// assertThat(unescaped2).isEqualTo("' one ' two '");
// }
//
// @Test
// public void testBackslashUnescaping() {
// String escaped="front \\\\\" back";
// LongFunction<String> func = String::valueOf;
// Template template = new Template("{} extra", func);
// String unescaped = template.unescape(escaped);
// assertThat(unescaped).isEqualTo("front \\\" back");
//
// }
}