IntenseFinance/nosqlbench
3
0
Fork 0
mirror of https://github.com/nosqlbench/nosqlbench.git synced 2025-02-25 18:55:28 -06:00
Code Issues Packages Projects Releases Wiki Activity

incremental progress on docs system

Browse Source
This commit is contained in:
Jonathan Shook
2020-05-01 13:56:49 -05:00
parent 62bdb11f70
commit 96e88ee68d
22 changed files with 449 additions and 159 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
nb-api/src/main/java/io/nosqlbench/nb/api/content/NBIOWalker.java
View File

@@ -14,6 +14,7 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
public class NBIOWalker {
private final static Logger logger = LogManager.getLogger(NBIOWalker.class);
public static void walk(Path p, PathVisitor v) {

2
nb-api/src/main/java/io/nosqlbench/nb/api/markdown/aggregator/DocsForTestingOnly.java → nb-api/src/main/java/io/nosqlbench/nb/api/markdown/DocsForTestingOnly.java
View File

@@ -1,4 +1,4 @@
package io.nosqlbench.nb.api.markdown.aggregator;
package io.nosqlbench.nb.api.markdown;
import io.nosqlbench.nb.annotations.Service;
import io.nosqlbench.nb.api.markdown.providers.DocsRootDirectory;

43
nb-api/src/main/java/io/nosqlbench/nb/api/markdown/aggregator/CompositeMarkdownInfo.java Normal file
View File

@@ -0,0 +1,43 @@
package io.nosqlbench.nb.api.markdown.aggregator;
import io.nosqlbench.nb.api.markdown.types.FrontMatterInfo;
import io.nosqlbench.nb.api.markdown.types.MarkdownInfo;
import java.nio.file.Path;
import java.util.LinkedList;
import java.util.List;
public class CompositeMarkdownInfo implements MarkdownInfo {
private List<MarkdownInfo> elements = new LinkedList<>();
@Override
public Path getPath() {
return elements.get(0).getPath();
}
@Override
public String getBody() {
StringBuilder sb = new StringBuilder();
for (MarkdownInfo element : elements) {
sb.append(element.getBody());
}
return sb.toString();
}
@Override
public FrontMatterInfo getFrontmatter() {
// calculate included topics
return null;
}
@Override
public boolean hasAggregations() {
// was true, but now it is false after compositing
return false;
}
public <T extends MarkdownInfo> CompositeMarkdownInfo add(T element) {
elements.add(element);
return this;
}
}

21
nb-api/src/main/java/io/nosqlbench/nb/api/markdown/aggregator/ListSplitterWhyDoesJavaNotDoThisAlready.java Normal file
View File

@@ -0,0 +1,21 @@
package io.nosqlbench.nb.api.markdown.aggregator;
import java.util.ArrayList;
import java.util.List;
import java.util.ListIterator;
import java.util.function.Predicate;
public class ListSplitterWhyDoesJavaNotDoThisAlready {
public static <T> List<? extends T> partition(List<? extends T> source, Predicate<T> filterout) {
ArrayList<T> filtered = new ArrayList<>();
ListIterator<? extends T> it = source.listIterator();
while (it.hasNext()) {
T element = it.next();
if (filterout.test(element)) {
it.remove();
filtered.add(element);
}
}
return filtered;
}
}

97
nb-api/src/main/java/io/nosqlbench/nb/api/markdown/aggregator/MarkdownDocs.java
View File

@@ -1,18 +1,22 @@
package io.nosqlbench.nb.api.markdown.aggregator;
import com.vladsch.flexmark.ext.yaml.front.matter.AbstractYamlFrontMatterVisitor;
import com.vladsch.flexmark.ext.yaml.front.matter.YamlFrontMatterExtension;
import com.vladsch.flexmark.html2md.converter.FlexmarkHtmlConverter;
import com.vladsch.flexmark.util.ast.Document;
import io.nosqlbench.nb.api.content.Content;
import io.nosqlbench.nb.api.markdown.FlexParser;
import io.nosqlbench.nb.api.markdown.providers.RawMarkdownSources;
import io.nosqlbench.nb.api.markdown.types.Diagnostics;
import io.nosqlbench.nb.api.markdown.types.DocScope;
import io.nosqlbench.nb.api.markdown.types.FrontMatterInfo;
import io.nosqlbench.nb.api.markdown.types.MarkdownInfo;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.util.*;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
public class MarkdownDocs {
private final static Logger logger = LogManager.getLogger(MarkdownDocs.class);
public static List<MarkdownInfo> find(DocScope... scopes) {
return find(".*", scopes);
}
@@ -29,18 +33,85 @@ public class MarkdownDocs {
// Find all topics and aggregators
List<String> aggregators = new ArrayList<>();
List<MutableMarkdownInfo> markdownInfos = markdownContent
List<MarkdownInfo> markdownInfos = markdownContent
.stream()
.map(MutableMarkdownInfo::new)
.map(ParsedMarkdown::new)
.collect(Collectors.toList());
List<Set<String>> collect =
List<String> diagBuffer = new ArrayList<>();
markdownInfos.forEach(i -> Diagnostics.getDiagnostics(i,diagBuffer));
diagBuffer.forEach(logger::warn);
List<Set<String>> topicSets =
markdownInfos.stream().map(m -> m.getFrontmatter().getTopics()).collect(Collectors.toList());
// find non-glob topics
List<String> nonGlobTopics = markdownInfos.stream()
.map(MarkdownInfo::getFrontmatter)
.map(FrontMatterInfo::getTopics).flatMap(Collection::stream)
.filter(s -> !isPattern(s))
.collect(Collectors.toList());
List<? extends MarkdownInfo> markdownWithTopicGlobs =
ListSplitterWhyDoesJavaNotDoThisAlready.partition(markdownInfos, MarkdownInfo::hasTopicGlobs);
Collections.
for (MarkdownInfo markdownWithTopicGlob : markdownWithTopicGlobs) {
}
// Assign glob topics to non-glob topics that match
for (MarkdownInfo parsedMarkdown : markdownInfos) {
FrontMatterInfo fm = parsedMarkdown.getFrontmatter();
Set<String> topics = fm.getTopics();
Set<String> newTopics = new HashSet<>();
for (String topic : topics) {
if (isPattern(topic)) {
Pattern p = Pattern.compile(topic);
for (String nonGlobTopic : nonGlobTopics) {
if (p.matcher(nonGlobTopic).matches()) {
newTopics.add(topic);
}
}
} else {
newTopics.add(topic);
}
}
fm.setTopics(newTopics);
}
// create topic to content map
HashMap<String,List<ParsedMarkdown>> contentByTopic = new HashMap<>();
for (ParsedMarkdown parsedMarkdown : markdownInfos) {
for (String topic : parsedMarkdown.getFrontmatter().getTopics()) {
contentByTopic.computeIfAbsent(topic, t -> new ArrayList<>()).add(parsedMarkdown);
}
}
ListIterator<? extends MarkdownInfo> lit = markdownInfos.listIterator();
while (lit.hasNext()) {
MarkdownInfo mif = lit.next();
if (mif.hasAggregations()) {
lit.remove();
mif = new CompositeMarkdownInfo().add(mif);
lit.add(mif);
}
}
// combine aggregate targets
for (ParsedMarkdown parsedMarkdown : markdownInfos) {
List<Pattern> aggregations = parsedMarkdown.getFrontmatter().getAggregations();
if (aggregations.size()>0) {
for (Pattern aggregation : aggregations) {
}
}
}
// Assign glob topics
// Assign content aggregates
System.out.println("topics: " + collect);
System.out.println("topics: " + topicSets);
aggregated.addAll(markdownInfos);
return aggregated;
@@ -48,4 +119,12 @@ public class MarkdownDocs {
}
private static boolean isPattern(String srcTopic) {
return
srcTopic.startsWith("^")
|| srcTopic.contains(".*")
|| srcTopic.contains(".+")
|| srcTopic.endsWith("$");
}
}

11
nb-api/src/main/java/io/nosqlbench/nb/api/markdown/aggregator/MarkdownInfo.java
View File

@@ -1,11 +0,0 @@
package io.nosqlbench.nb.api.markdown.aggregator;
import java.nio.file.Path;
public interface MarkdownInfo {
Path getPath();
String getBody();
FrontMatter getFrontmatter();
}

73
nb-api/src/main/java/io/nosqlbench/nb/api/markdown/aggregator/MutableFrontMatter.java
View File

@@ -1,73 +0,0 @@
package io.nosqlbench.nb.api.markdown.aggregator;
import java.security.InvalidParameterException;
import java.util.*;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
public class MutableFrontMatter implements FrontMatter {
private final Map<String, List<String>> data;
public MutableFrontMatter(Map<String, List<String>> data) {
this.data = data;
}
@Override
public String getTitle() {
List<String> titles = data.get(FrontMatter.TITLE);
if (titles==null) {
return "";
}
if (titles.size()!=1) {
throw new InvalidParameterException(FrontMatter.TITLE + " can only contain a single value.");
}
return titles.get(0);
}
@Override
public int getWeight() {
List<String> weights = data.get(FrontMatter.WEIGHT);
if (weights==null) {
return 0;
}
if (weights.size()!=1) {
throw new InvalidParameterException(FrontMatter.WEIGHT + " can only contain a single value.");
}
return Integer.parseInt(weights.get(0));
}
@Override
public Set<String> getTopics() {
List<String> topics = data.get(FrontMatter.TOPICS);
List<String> topic = data.get(FrontMatter.TOPIC);
if (topics==null && topic==null) {
return Set.of();
}
Set<String> topicSet = new HashSet<>();
if (topics!=null) {
topicSet.addAll(topics);
}
if (topic!=null) {
topicSet.addAll(topic);
}
return topicSet;
}
@Override
public List<Pattern> getAggregations() {
if (!data.containsKey(FrontMatter.AGGREGATIONS)) {
return List.of();
}
return data.get(FrontMatter.AGGREGATIONS).stream().map(Pattern::compile).collect(Collectors.toList());
}
@Override
public Set<DocScope> getDocScopes() {
if (!data.containsKey(FrontMatter.SCOPE)) {
return Set.of(DocScope.NONE);
}
return data.get(FrontMatter.SCOPE).stream().map(DocScope::valueOf).collect(Collectors.toSet());
}
}

41
nb-api/src/main/java/io/nosqlbench/nb/api/markdown/aggregator/MutableMarkdownInfo.java
View File

@@ -1,41 +0,0 @@
package io.nosqlbench.nb.api.markdown.aggregator;
import com.vladsch.flexmark.ext.yaml.front.matter.AbstractYamlFrontMatterVisitor;
import com.vladsch.flexmark.util.ast.Document;
import io.nosqlbench.nb.api.content.Content;
import io.nosqlbench.nb.api.markdown.FlexParser;
import java.nio.file.Path;
import java.util.List;
import java.util.Map;
public class MutableMarkdownInfo implements MarkdownInfo {
private final FrontMatter frontMatter;
private final Content<?> content;
public MutableMarkdownInfo(Content<?> content) {
String rawMarkdown = content.asString();
AbstractYamlFrontMatterVisitor v = new AbstractYamlFrontMatterVisitor();
Document parsed = FlexParser.parser.parse(rawMarkdown);
v.visit(parsed);
Map<String, List<String>> data = v.getData();
frontMatter = new MutableFrontMatter(data);
this.content = content;
}
@Override
public Path getPath() {
return content.asPath();
}
@Override
public String getBody() {
return null;
}
@Override
public FrontMatter getFrontmatter() {
return frontMatter;
}
}

98
nb-api/src/main/java/io/nosqlbench/nb/api/markdown/aggregator/ParsedFrontMatter.java Normal file
View File

@@ -0,0 +1,98 @@
package io.nosqlbench.nb.api.markdown.aggregator;
import io.nosqlbench.nb.api.markdown.types.DocScope;
import io.nosqlbench.nb.api.markdown.types.FrontMatterInfo;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.security.InvalidParameterException;
import java.util.*;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
public class ParsedFrontMatter implements FrontMatterInfo {
private final static Logger logger = LogManager.getLogger(ParsedFrontMatter.class);
private final Map<String, List<String>> data;
public ParsedFrontMatter(Map<String, List<String>> data) {
this.data = data;
}
@Override
public String getTitle() {
List<String> titles = data.get(FrontMatterInfo.TITLE);
if (titles==null) {
return "";
}
if (titles.size()!=1) {
throw new InvalidParameterException(FrontMatterInfo.TITLE + " can only contain a single value.");
}
return titles.get(0);
}
@Override
public int getWeight() {
List<String> weights = data.get(FrontMatterInfo.WEIGHT);
if (weights==null) {
return 0;
}
if (weights.size()!=1) {
throw new InvalidParameterException(FrontMatterInfo.WEIGHT + " can only contain a single value.");
}
return Integer.parseInt(weights.get(0));
}
@Override
public Set<String> getTopics() {
List<String> topics = data.get(FrontMatterInfo.TOPICS);
Set<String> topicSet = new HashSet<>();
if (topics!=null) {
for (String topic : topics) {
Collections.addAll(topicSet, topic.split(", *"));
}
topicSet.addAll(topics);
}
return topicSet;
}
@Override
public List<Pattern> getAggregations() {
if (!data.containsKey(FrontMatterInfo.AGGREGATE)) {
return List.of();
}
List<String> patterns = new ArrayList<>();
for (String aggName : data.get(FrontMatterInfo.AGGREGATE)) {
Collections.addAll(patterns,aggName.split(", *"));
}
return patterns.stream().map(Pattern::compile).collect(Collectors.toList());
}
@Override
public Set<DocScope> getDocScopes() {
if (!data.containsKey(FrontMatterInfo.SCOPES)) {
return Set.of(DocScope.NONE);
}
List<String> scopeNames = new ArrayList<>();
for (String scopeName : data.get(FrontMatterInfo.SCOPES)) {
Collections.addAll(scopeNames,scopeName.split(", *"));
}
return scopeNames.stream().map(DocScope::valueOf).collect(Collectors.toSet());
}
public List<String> getDiagnostics() {
List<String> warnings = new ArrayList<>();
for (String propname : data.keySet()) {
if (!FrontMatterInfo.FrontMatterKeyWords.contains(propname)) {
warnings.add("unrecognized frontm atter property " + propname);
}
}
return warnings;
}
public void setTopics(Set<String> newTopics) {
this.data.put(FrontMatterInfo.TOPICS,newTopics);
}
}

85
nb-api/src/main/java/io/nosqlbench/nb/api/markdown/aggregator/ParsedMarkdown.java Normal file
View File

@@ -0,0 +1,85 @@
package io.nosqlbench.nb.api.markdown.aggregator;
import com.vladsch.flexmark.ext.yaml.front.matter.AbstractYamlFrontMatterVisitor;
import com.vladsch.flexmark.util.ast.Document;
import io.nosqlbench.nb.api.content.Content;
import io.nosqlbench.nb.api.markdown.FlexParser;
import io.nosqlbench.nb.api.markdown.types.FrontMatterInfo;
import io.nosqlbench.nb.api.markdown.types.HasDiagnostics;
import io.nosqlbench.nb.api.markdown.types.MarkdownInfo;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
/**
* TODO: Make this a value type
*/
public class ParsedMarkdown implements MarkdownInfo, HasDiagnostics {
private final static Logger logger = LogManager.getLogger(MarkdownDocs.class);
private final ParsedFrontMatter frontMatter;
private final Content<?> content;
public ParsedMarkdown(Content<?> content) {
String rawMarkdown = content.asString();
AbstractYamlFrontMatterVisitor v = new AbstractYamlFrontMatterVisitor();
Document parsed = FlexParser.parser.parse(rawMarkdown);
v.visit(parsed);
Map<String, List<String>> data = v.getData();
frontMatter = new ParsedFrontMatter(data);
this.content = content;
logger.debug("created " + this.toString());
}
@Override
public Path getPath() {
return content.asPath();
}
@Override
public String getBody() {
return null;
}
@Override
public FrontMatterInfo getFrontmatter() {
return frontMatter;
}
/**
* Get a list of diagnostic warnings that might help users know of issues in their
* markdown content before publication.
* @param buffer A buffer object, for accumulating many lines of detail, if necessary.
* @return The buffer, with possible additions
*/
@Override
public List<String> getDiagnostics(List<String> buffer) {
List<String> diagnostics = frontMatter.getDiagnostics();
if (diagnostics.size()==0) {
return List.of();
}
String[] diags = diagnostics.stream().map(s -> " " + s).toArray(String[]::new);
buffer.add("found " + diagnostics.size() + " diagnostics for " + getPath().toString());
buffer.addAll(Arrays.asList(diags));
return buffer;
}
/**
* The buffer-less version of {@link #getDiagnostics(List)}
* @return a list of diagnostics lines, zero if there are none
*/
@Override
public List<String> getDiagnostics() {
return getDiagnostics(new ArrayList<>());
}
@Override
public boolean hasAggregations() {
return getFrontmatter().getAggregations().size()>0;
}
}

4
nb-api/src/main/java/io/nosqlbench/nb/api/markdown/exporter/MarkdownExporter.java
View File

@@ -1,8 +1,8 @@
package io.nosqlbench.nb.api.markdown.exporter;
import io.nosqlbench.nb.api.markdown.aggregator.DocScope;
import io.nosqlbench.nb.api.markdown.types.DocScope;
import io.nosqlbench.nb.api.markdown.aggregator.MarkdownDocs;
import io.nosqlbench.nb.api.markdown.aggregator.MarkdownInfo;
import io.nosqlbench.nb.api.markdown.types.MarkdownInfo;
import joptsimple.*;
import java.nio.file.Path;

20
nb-api/src/main/java/io/nosqlbench/nb/api/markdown/types/Diagnostics.java Normal file
View File

@@ -0,0 +1,20 @@
package io.nosqlbench.nb.api.markdown.types;
import io.nosqlbench.nb.api.markdown.types.HasDiagnostics;
import java.util.List;
public class Diagnostics {
public static List<String> getDiagnostics(Object o) {
if (o instanceof HasDiagnostics) {
return ((HasDiagnostics)o).getDiagnostics();
}
return List.of();
}
public static List<String> getDiagnostics(Object o, List<String> buffer) {
if (o instanceof HasDiagnostics) {
return ((HasDiagnostics)o).getDiagnostics(buffer);
}
return buffer;
}
}

8
nb-api/src/main/java/io/nosqlbench/nb/api/markdown/aggregator/DocScope.java → nb-api/src/main/java/io/nosqlbench/nb/api/markdown/types/DocScope.java
View File

@@ -1,4 +1,4 @@
package io.nosqlbench.nb.api.markdown.aggregator;
package io.nosqlbench.nb.api.markdown.types;
/**
* DocScope determines which display mode a topic is meant to be displayed in.
@@ -13,19 +13,19 @@ public enum DocScope {
* The command line doc scope includes any markdown which should be shown to the user
* when they are searching for or viewing documentation on a command line.
*/
CommandLine(false),
cli(false),
/**
* The static web doc scope includes any markdown which should be shown to the user
* when they are viewing documentation on an externally hosted site in static form.
*/
StaticWeb(false),
web(false),
/**
* The dynamic web doc scope includes any markdown which should be made available to
* users when they are interacting with a web application.
*/
DynamicWeb(false),
app(false),
/**
* ANY is a descriptive doc scope which is meant to be used as a filter within API calls

10
nb-api/src/main/java/io/nosqlbench/nb/api/markdown/aggregator/FrontMatter.java → nb-api/src/main/java/io/nosqlbench/nb/api/markdown/types/FrontMatterInfo.java
View File

@@ -1,4 +1,4 @@
package io.nosqlbench.nb.api.markdown.aggregator;
package io.nosqlbench.nb.api.markdown.types;
import java.util.List;
import java.util.Set;
@@ -9,14 +9,14 @@ import java.util.regex.Pattern;
* If the markdown source file does not contain the metadata requested, then reasonable non-null
* defaults must be provided.
*/
public interface FrontMatter {
public interface FrontMatterInfo {
String SCOPE = "scope";
String AGGREGATIONS = "aggregations";
String SCOPES = "scopes";
String AGGREGATE = "aggregate";
String TOPICS = "topics";
String TOPIC = "topic";
String WEIGHT = "weight";
String TITLE = "title";
Set<String> FrontMatterKeyWords = Set.of(SCOPES, AGGREGATE,TOPICS,WEIGHT,TITLE);
/**

8
nb-api/src/main/java/io/nosqlbench/nb/api/markdown/types/HasDiagnostics.java Normal file
View File

@@ -0,0 +1,8 @@
package io.nosqlbench.nb.api.markdown.types;
import java.util.List;
public interface HasDiagnostics {
List<String> getDiagnostics(List<String> buffer);
List<String> getDiagnostics();
}

42
nb-api/src/main/java/io/nosqlbench/nb/api/markdown/types/MarkdownInfo.java Normal file
View File

@@ -0,0 +1,42 @@
package io.nosqlbench.nb.api.markdown.types;
import io.nosqlbench.nb.api.markdown.types.FrontMatterInfo;
import java.nio.file.Path;
import java.util.List;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
public interface MarkdownInfo {
Path getPath();
String getBody();
FrontMatterInfo getFrontmatter();
boolean hasAggregations();
default boolean hasTopicGlobs() {
return getTopicGlobs().size()>0;
}
default List<Pattern> getTopicGlobs() {
return getFrontmatter().getTopics().stream()
.filter(t -> t.startsWith("^") || t.endsWith("$") || t.contains(".*") || t.contains(".+"))
.map(Pattern::compile)
.collect(Collectors.toList());
}
default List<String> getTopics() {
return getFrontmatter().getTopics().stream()
.filter(t -> !t.startsWith("^") && !t.endsWith("$") && !t.contains(".*") && !t.contains(".+"))
.collect(Collectors.toList());
}
default boolean hasAggregators() {
return getFrontmatter().getAggregations().size()>0;
}
default List<Pattern> getAggregators() {
return getFrontmatter().getAggregations();
}
}

4
nb-api/src/main/resources/docs-for-testing-only/topics-a/entry1-1.md → nb-api/src/main/resources/docs-for-testing-only/srcmain-topics-a/srcmain-entry1-1.md
View File

@@ -1,10 +1,10 @@
---
title: Entry 1-1
title: srcmain-Entry 1-1
weight: 37
topics: entries/entry2-1, related-topic-for-entry1-1
aggregate: topic
---
# Title Heading for Entry 1-1
# Title Heading for srcmain-Entry 1-1

8
nb-api/src/main/resources/docs-for-testing-only/srcmain-topics-b/srcmain-topic2-1/srcmain-entry-2-1-L.md Normal file
View File

@@ -0,0 +1,8 @@
---
title: srcmain-Entry 2-1-L
weight: 39
---
# Title Heading for srcmain-Entry 2-1-L

2
nb-api/src/test/java/io/nosqlbench/nb/api/markdown/aggregator/MarkdownDocsTest.java
View File

@@ -1,6 +1,6 @@
package io.nosqlbench.nb.api.markdown.aggregator;
import org.junit.Ignore;
import io.nosqlbench.nb.api.markdown.types.MarkdownInfo;
import org.junit.Test;
import java.util.List;

12
nb-api/src/test/resources/docs-for-testing-only/srctest-topics-a/srctest-entry1-1.md Normal file
View File

@@ -0,0 +1,12 @@
---
title: srctest-Entry 1-1
weight: 37
topics:
- srctest-entries/entry1-1, srctest-related-topic-for-entry1-1
- srctest-entries/entry1-1-extended
aggregate: topic
---
# Title Heading for srctest-Entry 1-1

8
nb-api/src/test/resources/docs-for-testing-only/srctest-topics-b/intesting-topic2-1/intesting-entry-2-1-L.md Normal file
View File

@@ -0,0 +1,8 @@
---
title: srctest-Entry 2-1-L
weight: 39
---
# Title Heading for srctest-Entry 2-1-L

10
nb-api/src/test/resources/docs-for-testing-only/topics-a/entry1-1.md
View File

@@ -1,10 +0,0 @@
---
title: Entry 1-1
weight: 37
topic: entries/entry1-1
aggregate: topic
---
# Title Heading for Entry 1-1