IntenseFinance/nosqlbench
3
0
Fork 0
mirror of https://github.com/nosqlbench/nosqlbench.git synced 2025-02-25 18:55:28 -06:00
Code Issues Packages Projects Releases Wiki Activity

debugged broken template, improved diagnostics

Browse Source
This commit is contained in:
Jonathan Shook 2020-04-13 14:38:54 -05:00
parent 612413cf09
commit 2e62af2be6
5 changed files with 97 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
activitytype-stdout/src/main/java/io/nosqlbench/activitytype/stdout/StdoutActivity.java
View File

@ -138,13 +138,23 @@ public class StdoutActivity extends SimpleActivity implements ActivityDefObserve
String format = getParams().getOptionalString("format").orElse(null);
if ((stmts.size()==0 && stmtsDocList.getDocBindings().size() > 0) || format!=null) {
logger.info("Creating stdout statement template from bindings, since none is otherwise defined.");
String generatedStmt = genStatementTemplate(stmtsDocList.getDocBindings().keySet());
BindingsTemplate bt = new BindingsTemplate();
stmtsDocList.getDocBindings().forEach(bt::addFieldBinding);
StringBindingsTemplate sbt = new StringBindingsTemplate(generatedStmt, bt);
StringBindings sb = sbt.resolve();
sequencer.addOp(sb,1L);
if (format!=null && format.startsWith("diag")) {
logger.info("Creating diagnostic log for resolver construction...");
BindingsTemplate bt = new BindingsTemplate();
stmtsDocList.getDocBindings().forEach(bt::addFieldBinding);
String diagnostics = bt.getDiagnostics();
System.out.println(diagnostics);
System.out.flush();
System.exit(2);
} else {
logger.info("Creating stdout statement template from bindings, since none is otherwise defined.");
String generatedStmt = genStatementTemplate(stmtsDocList.getDocBindings().keySet());
BindingsTemplate bt = new BindingsTemplate();
stmtsDocList.getDocBindings().forEach(bt::addFieldBinding);
StringBindingsTemplate sbt = new StringBindingsTemplate(generatedStmt, bt);
StringBindings sb = sbt.resolve();
sequencer.addOp(sb,1L);
}
} else if (stmts.size() > 0) {
for (StmtDef stmt : stmts) {
ParsedStmt parsed = stmt.getParsed().orError();

8
activitytype-stdout/src/main/resources/stdout.md
View File

@ -33,9 +33,11 @@ activity types.
- **newline** - whether to automatically add a missing newline to the end
of any statements.
default: true
- **format** - which format to use. If provided, the format will override
any statement formats provided by the YAML.
valid values are (csv, readout, json, inlinejson, and assignments)
- **format** - which format to use. If provided, the format will override any statement formats provided by the YAML.
valid values are (csv, readout, json, inlinejson, assignments, and diag)
- When 'format=diag', then the internal construction logic for the binding is logged in detail and nosqlbench exits.
This is useful for detailed diagnostics when you run into trouble, but not generally otherwise. This provides
details that you may include in a bug report if you think there is a bindings bug.
## Configuration

41
virtdata-api/src/main/java/io/nosqlbench/virtdata/core/bindings/BindingsTemplate.java
View File

@ -38,7 +38,7 @@ import java.util.Optional;
* bindings will be used in.
*/
public class BindingsTemplate {
private final static Logger logger = LogManager.getLogger(BindingsTemplate.class);
private final static Logger logger = LogManager.getLogger(BindingsTemplate.class);
private List<String> bindPointNames = new ArrayList<>();
private List<String> specifiers = new ArrayList<>();
@ -47,11 +47,11 @@ public class BindingsTemplate {
// }
public BindingsTemplate(List<String> anchors, List<String> specs) {
if (anchors.size()!=specs.size()) {
if (anchors.size() != specs.size()) {
throw new InvalidParameterException("Anchors and Specifiers must be matched pair-wise.");
}
for (int i = 0; i < anchors.size(); i++) {
addFieldBinding(anchors.get(i),specs.get(i));
addFieldBinding(anchors.get(i), specs.get(i));
}
}
@ -64,14 +64,15 @@ public class BindingsTemplate {
public void addFieldBindings(List<BindPoint> bindPoints) {
for (BindPoint bindPoint : bindPoints) {
addFieldBinding(bindPoint.getAnchor(),bindPoint.getBindspec());
addFieldBinding(bindPoint.getAnchor(), bindPoint.getBindspec());
}
}
/**
* Add a named binding specifier to the template
*
* @param bindPointName the name associated with the binding specifier
* @param genSpec the binding specifier
* @param genSpec the binding specifier
*/
public void addFieldBinding(String bindPointName, String genSpec) {
this.bindPointNames.add(bindPointName);
@ -80,19 +81,39 @@ public class BindingsTemplate {
/**
* Add multiple named bindings to the template
*
* @param bindPairs A map of named binding specifiers
*/
public void addFieldBindings(Map<String,String> bindPairs) {
public void addFieldBindings(Map<String, String> bindPairs) {
for (Map.Entry<String, String> e : bindPairs.entrySet()) {
this.bindPointNames.add(e.getKey());
this.specifiers.add(e.getValue());
}
}
public String getDiagnostics() {
StringBuilder diaglog = new StringBuilder();
for (String specifier : specifiers) {
diaglog.append("for ").append(specifier).append(":");
ResolverDiagnostics mapperDiagnostics = VirtData.getMapperDiagnostics(specifier);
String diagnostics = mapperDiagnostics.toString();
diaglog.append(diagnostics);
if (mapperDiagnostics.getResolvedFunction().isPresent()) {
diaglog.append("☑ RESOLVED:")
.append(mapperDiagnostics.getResolvedFunction().get().toString()).append("\n");
} else {
diaglog.append("☐ UNRESOLVED\n");
}
}
return diaglog.toString();
}
/**
* Use the data mapping library and the specifier to create instances of data mapping functions.
* If you need thread-aware mapping, be sure to call this in the proper thread. Each time this method
* is called, it creates a new instance.
*
* @return A set of bindings that can be used to yield mapped data values later.
*/
public Bindings resolveBindings() {
@ -104,9 +125,9 @@ public class BindingsTemplate {
} else {
logAvailableDataMappers();
throw new RuntimeException(
"data mapper binding was unsuccessful for "
+ ", spec:" + specifier
+ ", see log for known data mapper names.");
"data mapper binding was unsuccessful for "
+ ", spec:" + specifier
+ ", see log for known data mapper names.");
}
}
return new Bindings(this, dataMappers);
@ -149,7 +170,7 @@ public class BindingsTemplate {
sb.append("=>[");
sb.append(values[i]);
sb.append("](");
sb.append((null!=values[i]) ? values[i].getClass().getSimpleName() : "NULL");
sb.append((null != values[i]) ? values[i].getClass().getSimpleName() : "NULL");
sb.append(")");
delim = ", ";
}

64
virtdata-api/src/main/java/io/nosqlbench/virtdata/core/bindings/VirtDataComposer.java
View File

@ -50,11 +50,11 @@ import java.util.stream.Collectors;
public class VirtDataComposer {
private final static String PREAMBLE = "compose ";
private final static Logger logger = LogManager.getLogger(DataMapperLibrary.class);
private final static Logger logger = LogManager.getLogger(DataMapperLibrary.class);
private final static MethodHandles.Lookup lookup = MethodHandles.publicLookup();
private final VirtDataFunctionLibrary functionLibrary;
private final Map<String,Object> customElements = new HashMap<>();
private final Map<String, Object> customElements = new HashMap<>();
public VirtDataComposer(VirtDataFunctionLibrary functionLibrary) {
this.functionLibrary = functionLibrary;
@ -90,23 +90,22 @@ public class VirtDataComposer {
public ResolverDiagnostics resolveDiagnosticFunctionFlow(VirtDataFlow flow) {
ResolverDiagnostics diagnostics = new ResolverDiagnostics();
diagnostics.trace("processing flow " + flow.toString() + " from output to input");
LinkedList<List<ResolvedFunction>> funcs = new LinkedList<>();
LinkedList<Set<Class<?>>> nextFunctionInputTypes = new LinkedList<>();
Optional<Class<?>> finalValueTypeOption =
Optional.ofNullable(flow.getLastExpression().getCall().getOutputType())
.map(ValueType::valueOfClassName).map(ValueType::getValueClass);
Optional.ofNullable(flow.getLastExpression().getCall().getOutputType())
.map(ValueType::valueOfClassName).map(ValueType::getValueClass);
nextFunctionInputTypes.add(new HashSet<>());
finalValueTypeOption.ifPresent(t -> nextFunctionInputTypes.get(0).add(t));
diagnostics.trace("working backwards from " + (flow.getExpressions().size()-1));
diagnostics.trace("working backwards from index " + (flow.getExpressions().size() - 1) + " to index 0");
for (int i = flow.getExpressions().size() - 1; i >= 0; i--) {
FunctionCall call = flow.getExpressions().get(i).getCall();
diagnostics.trace("resolving args for " + call.toString());
diagnostics.trace("FUNCTION[" + i + "]: " + call.toString() + ", resolving args");
// diagnostics.trace("resolving args for " + call.toString());
List<ResolvedFunction> nodeFunctions = new LinkedList<>();
@ -114,23 +113,24 @@ public class VirtDataComposer {
Class<?> inputType = ValueType.classOfType(call.getInputType());
Class<?> outputType = ValueType.classOfType(call.getOutputType());
Object[] args = call.getArguments();
try {
args = populateFunctions(diagnostics, args, this.customElements);
} catch (Exception e) {
return diagnostics.error(e);
}
diagnostics.trace("resolved args: ");
diagnostics.trace(" resolved args:");
for (Object arg : args) {
diagnostics.trace(" " + arg.getClass().getSimpleName() + ": " + arg.toString());
diagnostics.trace(" " + arg.getClass().getSimpleName() + ": " + arg.getClass().getCanonicalName());
}
List<ResolvedFunction> resolved = functionLibrary.resolveFunctions(outputType, inputType, funcName, this.customElements,args);
List<ResolvedFunction> resolved = functionLibrary.resolveFunctions(outputType, inputType, funcName, this.customElements, args);
if (resolved.size() == 0) {
return diagnostics.error(new RuntimeException("Unable to find even one function for " + call));
}
diagnostics.trace(" resolved functions:");
diagnostics.trace(summarize(resolved));
diagnostics.trace(" resolved functions");
diagnostics.trace(summarize(resolved, " - "));
nodeFunctions.addAll(resolved);
funcs.addFirst(nodeFunctions);
@ -152,10 +152,10 @@ public class VirtDataComposer {
}
FunctionAssembly assembly = new FunctionAssembly();
diagnostics.trace("composed summary: " + summarize(flattenedFuncs));
// diagnostics.trace("composed summary: " + summarize(flattenedFuncs));
boolean isThreadSafe = true;
diagnostics.trace("FUNCTION chain selected: (multi) '" + this.summarize(flattenedFuncs) + "'");
diagnostics.trace("FUNCTION chain selected: (multi) '" + this.summarize(flattenedFuncs, " - ") + "'");
for (ResolvedFunction resolvedFunction : flattenedFuncs) {
try {
Object functionObject = resolvedFunction.getFunctionObject();
@ -164,7 +164,7 @@ public class VirtDataComposer {
isThreadSafe = false;
}
} catch (Exception e) {
String flowdata = flow!=null? flow.toString() : "undefined";
String flowdata = flow != null ? flow.toString() : "undefined";
return diagnostics.error(new RuntimeException("FUNCTION resolution failed: '" + flowdata + "': " + e.toString()));
}
}
@ -178,7 +178,7 @@ public class VirtDataComposer {
return resolverDiagnostics.getResolvedFunction();
}
private Object[] populateFunctions(ResolverDiagnostics diagnostics, Object[] args, Map<String,?> cconfig) {
private Object[] populateFunctions(ResolverDiagnostics diagnostics, Object[] args, Map<String, ?> cconfig) {
for (int i = 0; i < args.length; i++) {
Object o = args[i];
if (o instanceof FunctionCall) {
@ -188,7 +188,8 @@ public class VirtDataComposer {
Class<?> inputType = ValueType.classOfType(call.getInputType());
Class<?> outputType = ValueType.classOfType(call.getOutputType());
Object[] fargs = call.getArguments();
diagnostics.trace("resolving argument as function '" + call.toString() + "'");
diagnostics.trace(" arg (function): " + call.toString());
// diagnostics.trace("resolving argument as function '" + call.toString() + "'");
fargs = populateFunctions(diagnostics, fargs, cconfig);
List<ResolvedFunction> resolved = functionLibrary.resolveFunctions(outputType, inputType, funcName, cconfig, fargs);
@ -215,9 +216,11 @@ public class VirtDataComposer {
funcs.removeAll(toRemove);
}
private String summarize(List<ResolvedFunction> funcs) {
private String summarize(List<ResolvedFunction> funcs, String prefix) {
return funcs.stream()
.map(String::valueOf).collect(Collectors.joining("|"));
.map(String::valueOf)
.map(s -> prefix + s)
.collect(Collectors.joining("\n"));
}
private String summarizeBulk(List<List<ResolvedFunction>> funcs) {
@ -225,9 +228,9 @@ public class VirtDataComposer {
List<List<String>> spans = new LinkedList<>();
funcs.forEach(l -> spans.add(l.stream().map(String::valueOf).collect(Collectors.toList())));
List<Optional<Integer>> widths = spans.stream().map(
l -> l.stream().map(String::length).max(Integer::compare)).collect(Collectors.toList());
l -> l.stream().map(String::length).max(Integer::compare)).collect(Collectors.toList());
String funcsdata = spans.stream().map(
l -> l.stream().map(String::valueOf).collect(Collectors.joining("|\n"))
l -> l.stream().map(String::valueOf).collect(Collectors.joining("|\n"))
).collect(Collectors.joining("\n\n"));
StringBuilder sb = new StringBuilder();
@ -321,7 +324,7 @@ public class VirtDataComposer {
funcs.sort(ResolvedFunction.PREFERRED_TYPE_COMPARATOR);
while (funcs.size() > 1) {
logger.trace("BY-SINGLE-PREFERRED-TYPE removing func " + funcs.get(funcs.size() - 1)
+ " because " + funcs.get(0) + " has more preferred types.");
+ " because " + funcs.get(0) + " has more preferred types.");
funcs.remove(funcs.size() - 1);
}
@ -336,7 +339,7 @@ public class VirtDataComposer {
prevFuncs.sort(ResolvedFunction.PREFERRED_TYPE_COMPARATOR);
while (prevFuncs.size() > 1) {
String logmsg = "BY-PREV-PREFERRED-TYPE removing func " + prevFuncs.get(prevFuncs.size() - 1)
+ " because " + prevFuncs.get(0) + " has more preferred types.";
+ " because " + prevFuncs.get(0) + " has more preferred types.";
logger.trace(logmsg);
prevFuncs.remove(prevFuncs.size() - 1);
}
@ -345,7 +348,7 @@ public class VirtDataComposer {
nextFuncs.sort(ResolvedFunction.PREFERRED_TYPE_COMPARATOR);
while (nextFuncs.size() > 1) {
String logmsg = "BY-NEXT-PREFERRED-TYPE removing func " + nextFuncs.get(nextFuncs.size() - 1)
+ " because " + nextFuncs.get(0) + " has more preferred types.";
+ " because " + nextFuncs.get(0) + " has more preferred types.";
logger.trace(logmsg);
nextFuncs.remove(nextFuncs.size() - 1);
}
@ -369,7 +372,7 @@ public class VirtDataComposer {
Set<Class<?>> outputs = getOutputs(prevFuncs);
Set<Class<?>> inputs = getInputs(nextFuncs);
Set<Class<?>> directMatches =
inputs.stream().filter(outputs::contains).collect(Collectors.toCollection(HashSet::new));
inputs.stream().filter(outputs::contains).collect(Collectors.toCollection(HashSet::new));
if (directMatches.size() > 0) {
List<ResolvedFunction> toremove = new ArrayList<>();
@ -423,9 +426,9 @@ public class VirtDataComposer {
return 0;
} else {
toremove.forEach(nextfunc -> {
String logmsg = "BY-ASSIGNABLE-TYPE removing next func: " + nextfunc + " because its input types are not assignable from any of the previous funcs";
logger.trace(logmsg);
}
String logmsg = "BY-ASSIGNABLE-TYPE removing next func: " + nextfunc + " because its input types are not assignable from any of the previous funcs";
logger.trace(logmsg);
}
);
nextFuncs.removeAll(toremove);
@ -450,9 +453,10 @@ public class VirtDataComposer {
return inputs;
}
public Map<String,?> getCustomElements() {
public Map<String, ?> getCustomElements() {
return this.customElements;
}
public VirtDataComposer addCustomElement(String name, Object element) {
this.customElements.put(name, element);
return this;

10
virtdata-api/src/main/java/io/nosqlbench/virtdata/core/templates/StringBindingsTemplate.java
View File

@ -34,4 +34,14 @@ public class StringBindingsTemplate {
Bindings bindings = bindingsTemplate.resolveBindings();
return new StringBindings(compositor,bindings);
}
public String getDiagnostics() {
StringCompositor compositor = new StringCompositor(stringTemplate);
HashSet<String> unqualifiedNames = new HashSet<>(compositor.getBindPointNames());
unqualifiedNames.removeAll(new HashSet<>(bindingsTemplate.getBindPointNames()));
if (unqualifiedNames.size()>0) {
throw new RuntimeException("Named anchors were specified in the template which were not provided in the bindings: " + unqualifiedNames.toString());
}
return bindingsTemplate.getDiagnostics();
}
}