IntenseFinance/nosqlbench
3
0
Fork 0
mirror of https://github.com/nosqlbench/nosqlbench.git synced 2025-02-25 18:55:28 -06:00
Code Issues Packages Projects Releases Wiki Activity

docs system updates

Browse Source
This commit is contained in:
Jonathan Shook 2020-04-30 10:39:37 -05:00
parent 8f213efc13
commit 7a454c8e1c
18 changed files with 324 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
engine-cli/src/main/java/io/nosqlbench/engine/cli/NBCLI.java
View File

@ -17,6 +17,7 @@ import io.nosqlbench.engine.core.metrics.MetricReporters;
import io.nosqlbench.engine.core.script.MetricsMapper;
import io.nosqlbench.engine.core.script.Scenario;
import io.nosqlbench.engine.core.script.ScenariosExecutor;
import io.nosqlbench.nb.api.markdown.exporter.MarkdownExporter;
import io.nosqlbench.virtdata.userlibs.apps.VirtDataMainApp;
import io.nosqlbench.docsys.core.DocServerApp;
import org.slf4j.Logger;
@ -70,6 +71,10 @@ public class NBCLI {
DocServerApp.main(Arrays.copyOfRange(args, 1, args.length));
System.exit(0);
}
if (args.length>0 && args[0].toLowerCase().equals(MarkdownExporter.APP_NAME)) {
MarkdownExporter.main(Arrays.copyOfRange(args,1,args.length));
System.exit(0);
}
NBCLIOptions options = new NBCLIOptions(args);
ConsoleLogging.enableConsoleLogging(options.wantsConsoleLogLevel(), options.getConsoleLoggingPattern());

18
engine-docs/src/main/java/io/nosqlbench/engine/docs/NosqlBenchRawMarkdownSource.java Normal file
View File

@ -0,0 +1,18 @@
package io.nosqlbench.engine.docs;
import io.nosqlbench.nb.annotations.Service;
import io.nosqlbench.nb.api.content.Content;
import io.nosqlbench.nb.api.markdown.providers.DocsRootDirectory;
import io.nosqlbench.nb.api.markdown.providers.RawMarkdownSource;
import java.util.List;
@Service(RawMarkdownSource.class)
public class NosqlBenchRawMarkdownSource extends DocsRootDirectory {
@Override
protected String getRootPathName() {
return "docs-for-eb";
}
}

19
nb-api/pom.xml
View File

@ -63,6 +63,11 @@
<artifactId>log4j-core</artifactId>
</dependency>
<dependency>
<groupId>net.sf.jopt-simple</groupId>
<artifactId>jopt-simple</artifactId>
<version>5.0.3</version>
</dependency>
<!-- perf testing -->
@ -97,4 +102,18 @@
</dependencies>
<build>
<resources>
<resource>
<directory>src/main/resources</directory>
<excludes>
<exclude>docs-for-testing-only/**</exclude>
</excludes>
<includes>
<include>**</include>
</includes>
</resource>
</resources>
</build>
</project>

28
nb-api/src/main/java/io/nosqlbench/nb/api/markdown/aggregator/DocScope.java
View File

@ -9,10 +9,37 @@ package io.nosqlbench.nb.api.markdown.aggregator;
*/
public enum DocScope {
/**
* The command line doc scope includes any markdown which should be shown to the user
* when they are searching for or viewing documentation on a command line.
*/
CommandLine(false),
/**
* The static web doc scope includes any markdown which should be shown to the user
* when they are viewing documentation on an externally hosted site in static form.
*/
StaticWeb(false),
/**
* The dynamic web doc scope includes any markdown which should be made available to
* users when they are interacting with a web application.
*/
DynamicWeb(false),
/**
* ANY is a descriptive doc scope which is meant to be used as a filter within API calls
* to find and display docs. It is invalid for any raw markdown content to be tagged
* with this doc scope.
*/
ANY(true),
/**
* NONE is a descriptive doc scope which is meant to be used as the default value for
* found raw markdown if it has not been tagged with a direct scope. This scope should only
* be returned as a place holder when no scopes are defined on content. When reading raw
* content, finding the NONE scope explicitly on raw source is considered an error.
*/
NONE(true);
/**
@ -20,7 +47,6 @@ public enum DocScope {
* as a default or qualifier, but should not be assigned in content metadata.
* Content readers should throw an error when ANY or NONE are found in raw content.
* Content readers should add ANY to any content which contains any non-query scope.
* Content readers should add NONE to any content which contains no non-query scope.
*
* This is added to provide a uniform and simple query interface.
*/

15
nb-api/src/main/java/io/nosqlbench/nb/api/markdown/aggregator/DocsForTestingOnly.java Normal file
View File

@ -0,0 +1,15 @@
package io.nosqlbench.nb.api.markdown.aggregator;
import io.nosqlbench.nb.annotations.Service;
import io.nosqlbench.nb.api.markdown.providers.DocsRootDirectory;
import io.nosqlbench.nb.api.markdown.providers.RawMarkdownSource;
@Service(RawMarkdownSource.class)
public class DocsForTestingOnly extends DocsRootDirectory {
@Override
protected String getRootPathName() {
return "docs-for-testing-only";
}
}

11
nb-api/src/main/java/io/nosqlbench/nb/api/markdown/aggregator/FrontMatter.java
View File

@ -10,6 +10,15 @@ import java.util.regex.Pattern;
* defaults must be provided.
*/
public interface FrontMatter {
String SCOPE = "scope";
String AGGREGATIONS = "aggregations";
String TOPICS = "topics";
String TOPIC = "topic";
String WEIGHT = "weight";
String TITLE = "title";
/**
* @return A title for the given markdown source file.
*/
@ -39,7 +48,7 @@ public interface FrontMatter {
*
* @return A list of categories
*/
List<String> getTopics();
Set<String> getTopics();
/**
* <p>

47
nb-api/src/main/java/io/nosqlbench/nb/api/markdown/aggregator/MarkdownDocs.java
View File

@ -3,54 +3,49 @@ package io.nosqlbench.nb.api.markdown.aggregator;
import com.vladsch.flexmark.ext.yaml.front.matter.AbstractYamlFrontMatterVisitor;
import com.vladsch.flexmark.ext.yaml.front.matter.YamlFrontMatterExtension;
import com.vladsch.flexmark.html2md.converter.FlexmarkHtmlConverter;
import com.vladsch.flexmark.parser.Parser;
import com.vladsch.flexmark.util.ast.Document;
import io.nosqlbench.nb.api.content.Content;
import io.nosqlbench.nb.api.markdown.FlexParser;
import io.nosqlbench.nb.api.markdown.providers.MarkdownProviders;
import io.nosqlbench.nb.api.markdown.providers.RawMarkdownSources;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.*;
import java.util.stream.Collectors;
public class MarkdownDocs {
private static FlexmarkHtmlConverter converter = FlexmarkHtmlConverter.builder()
.extensions(List.of(YamlFrontMatterExtension.create()))
.build();
public static List<MarkdownInfo> find(DocScope... scopes) {
return find(".*", scopes);
}
public static MarkdownInfo find(DocScope... scopes) {
return null;
public static List<MarkdownInfo> findAll() {
return find(DocScope.ANY);
}
public static List<MarkdownInfo> find(String name, DocScope... scopes) {
List<MarkdownInfo> aggregated = new ArrayList<>();
List<Content<?>> markdownContent = MarkdownProviders.getAllMarkdown();
MarkdownInfo info = new MutableMarkdownInfo();
List<Content<?>> markdownContent = RawMarkdownSources.getAllMarkdown();
// Find all topics and aggregators
List<String> aggregators = new ArrayList<>();
List<String> topics = new ArrayList<>();
for (Content<?> content : markdownContent) {
String markdown = content.asString();
String convert = converter.convert(markdown);
List<MutableMarkdownInfo> markdownInfos = markdownContent
.stream()
.map(MutableMarkdownInfo::new)
.collect(Collectors.toList());
AbstractYamlFrontMatterVisitor v = new AbstractYamlFrontMatterVisitor();
Document parsed = FlexParser.parser.parse(markdown);
List<Set<String>> collect =
markdownInfos.stream().map(m -> m.getFrontmatter().getTopics()).collect(Collectors.toList());
v.visit(parsed);
Map<String, List<String>> data = v.getData();
System.out.print("frontmatter for " + content.asPath());
for (Map.Entry<String, List<String>> e : data.entrySet()) {
System.out.println("k : " + e.getKey());
System.out.println(" v:" + String.join(",",e.getValue()));
}
}
// Assign glob topics
// Assign content aggregates
System.out.println("topics: " + collect);
aggregated.addAll(markdownInfos);
return aggregated;
}
}

1
nb-api/src/main/java/io/nosqlbench/nb/api/markdown/aggregator/MarkdownInfo.java
View File

@ -3,6 +3,7 @@ package io.nosqlbench.nb.api.markdown.aggregator;
import java.nio.file.Path;
public interface MarkdownInfo {
Path getPath();
String getBody();
FrontMatter getFrontmatter();

73
nb-api/src/main/java/io/nosqlbench/nb/api/markdown/aggregator/MutableFrontMatter.java Normal file
View File

@ -0,0 +1,73 @@
package io.nosqlbench.nb.api.markdown.aggregator;
import java.security.InvalidParameterException;
import java.util.*;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
public class MutableFrontMatter implements FrontMatter {
private final Map<String, List<String>> data;
public MutableFrontMatter(Map<String, List<String>> data) {
this.data = data;
}
@Override
public String getTitle() {
List<String> titles = data.get(FrontMatter.TITLE);
if (titles==null) {
return "";
}
if (titles.size()!=1) {
throw new InvalidParameterException(FrontMatter.TITLE + " can only contain a single value.");
}
return titles.get(0);
}
@Override
public int getWeight() {
List<String> weights = data.get(FrontMatter.WEIGHT);
if (weights==null) {
return 0;
}
if (weights.size()!=1) {
throw new InvalidParameterException(FrontMatter.WEIGHT + " can only contain a single value.");
}
return Integer.parseInt(weights.get(0));
}
@Override
public Set<String> getTopics() {
List<String> topics = data.get(FrontMatter.TOPICS);
List<String> topic = data.get(FrontMatter.TOPIC);
if (topics==null && topic==null) {
return Set.of();
}
Set<String> topicSet = new HashSet<>();
if (topics!=null) {
topicSet.addAll(topics);
}
if (topic!=null) {
topicSet.addAll(topic);
}
return topicSet;
}
@Override
public List<Pattern> getAggregations() {
if (!data.containsKey(FrontMatter.AGGREGATIONS)) {
return List.of();
}
return data.get(FrontMatter.AGGREGATIONS).stream().map(Pattern::compile).collect(Collectors.toList());
}
@Override
public Set<DocScope> getDocScopes() {
if (!data.containsKey(FrontMatter.SCOPE)) {
return Set.of(DocScope.NONE);
}
return data.get(FrontMatter.SCOPE).stream().map(DocScope::valueOf).collect(Collectors.toSet());
}
}

25
nb-api/src/main/java/io/nosqlbench/nb/api/markdown/aggregator/MutableMarkdownInfo.java
View File

@ -1,11 +1,32 @@
package io.nosqlbench.nb.api.markdown.aggregator;
import com.vladsch.flexmark.ext.yaml.front.matter.AbstractYamlFrontMatterVisitor;
import com.vladsch.flexmark.util.ast.Document;
import io.nosqlbench.nb.api.content.Content;
import io.nosqlbench.nb.api.markdown.FlexParser;
import java.nio.file.Path;
import java.util.List;
import java.util.Map;
public class MutableMarkdownInfo implements MarkdownInfo {
private final FrontMatter frontMatter;
private final Content<?> content;
public MutableMarkdownInfo(Content<?> content) {
String rawMarkdown = content.asString();
AbstractYamlFrontMatterVisitor v = new AbstractYamlFrontMatterVisitor();
Document parsed = FlexParser.parser.parse(rawMarkdown);
v.visit(parsed);
Map<String, List<String>> data = v.getData();
frontMatter = new MutableFrontMatter(data);
this.content = content;
}
@Override
public Path getPath() {
return null;
return content.asPath();
}
@Override
@ -15,6 +36,6 @@ public class MutableMarkdownInfo implements MarkdownInfo {
@Override
public FrontMatter getFrontmatter() {
return null;
return frontMatter;
}
}

52
nb-api/src/main/java/io/nosqlbench/nb/api/markdown/exporter/MarkdownExporter.java Normal file
View File

@ -0,0 +1,52 @@
package io.nosqlbench.nb.api.markdown.exporter;
import io.nosqlbench.nb.api.markdown.aggregator.DocScope;
import io.nosqlbench.nb.api.markdown.aggregator.MarkdownDocs;
import io.nosqlbench.nb.api.markdown.aggregator.MarkdownInfo;
import joptsimple.*;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
public class MarkdownExporter implements Runnable {
public static final String APP_NAME = "exporter";
private final Path basePath;
private Set<DocScope> scopeSet;
public MarkdownExporter(Path basePath, Set<DocScope> scopeSet) {
this.basePath = basePath;
this.scopeSet = scopeSet;
}
public static void main(String[] args) {
final OptionParser parser = new OptionParser();
OptionSpec<String> basedir = parser.accepts("basedir", "base directory to write to")
.withRequiredArg().ofType(String.class).defaultsTo(".");
OptionSpec<String> docScopes = parser.accepts("scopes", "scopes of documentation to export")
.withRequiredArg().ofType(String.class).defaultsTo(DocScope.ANY.toString());
parser.acceptsAll(List.of("-h","--help","help"),"Display help").forHelp();
OptionSet options = parser.parse(args);
Path basePath = Path.of(basedir.value(options));
Set<DocScope> scopeSet = docScopes.values(options).stream().map(DocScope::valueOf).collect(Collectors.toSet());
new MarkdownExporter(basePath,scopeSet).run();
}
@Override
public void run() {
List<MarkdownInfo> markdownInfos = MarkdownDocs.find(new ArrayList<>(scopeSet).toArray(new DocScope[0]));
}
}

22
engine-docs/src/main/java/io/nosqlbench/engine/docs/NosqlBenchMarkdownSource.java → nb-api/src/main/java/io/nosqlbench/nb/api/markdown/providers/DocsRootDirectory.java
View File

@ -1,30 +1,25 @@
package io.nosqlbench.engine.docs;
package io.nosqlbench.nb.api.markdown.providers;
import io.nosqlbench.docsys.api.Docs;
import io.nosqlbench.docsys.api.DocsBinder;
import io.nosqlbench.nb.api.annotations.Service;
import io.nosqlbench.nb.api.content.Content;
import io.nosqlbench.nb.api.content.NBIO;
import io.nosqlbench.nb.api.content.NBIOWalker;
import io.nosqlbench.nb.api.content.PathContent;
import io.nosqlbench.nb.api.markdown.providers.MarkdownProvider;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;
@Service(MarkdownProvider.class)
public class NosqlBenchMarkdownSource implements MarkdownProvider {
public abstract class DocsRootDirectory implements RawMarkdownSource {
@Override
public List<Content<?>> getMarkdownInfo() {
Path docspath = NBIO.local().name("docs-for-eb")
.one().asPath();
List<Content<?>> list = NBIO.local().name(getRootPathName()).list();
NBIOWalker.CollectVisitor capture = new NBIOWalker.CollectVisitor(true, false);
NBIOWalker.RegexFilter filter = new NBIOWalker.RegexFilter("\\.md",true);
NBIOWalker.walkFullPath(docspath,capture,filter);
for (Content<?> content : list) {
Path path = content.asPath();
NBIOWalker.walkFullPath(path,capture,filter);
}
List<Content<?>> content = new ArrayList<>();
for (Path path : capture.get()) {
content.add(new PathContent(path));
@ -32,4 +27,7 @@ public class NosqlBenchMarkdownSource implements MarkdownProvider {
return content;
}
protected abstract String getRootPathName();
}

20
nb-api/src/main/java/io/nosqlbench/nb/api/markdown/providers/MarkdownProviders.java
View File

@ -1,20 +0,0 @@
package io.nosqlbench.nb.api.markdown.providers;
import io.nosqlbench.nb.api.content.Content;
import java.util.ArrayList;
import java.util.List;
import java.util.ServiceLoader;
/**
* The internal
*/
public class MarkdownProviders {
public static List<Content<?>> getAllMarkdown() {
ServiceLoader<MarkdownProvider> loader = ServiceLoader.load(MarkdownProvider.class);
List<Content<?>> names = new ArrayList<>();
loader.iterator().forEachRemaining(d -> names.addAll(d.getMarkdownInfo()));
return names;
}
}

2
nb-api/src/main/java/io/nosqlbench/nb/api/markdown/providers/MarkdownProvider.java → nb-api/src/main/java/io/nosqlbench/nb/api/markdown/providers/RawMarkdownSource.java
View File

@ -7,6 +7,6 @@ import java.util.List;
/**
* A MarkdownProvider simply provides all the markdown content it is aware of.
*/
public interface MarkdownProvider {
public interface RawMarkdownSource {
List<Content<?>> getMarkdownInfo();
}

20
nb-api/src/main/java/io/nosqlbench/nb/api/markdown/providers/RawMarkdownSources.java Normal file
View File

@ -0,0 +1,20 @@
package io.nosqlbench.nb.api.markdown.providers;
import io.nosqlbench.nb.api.content.Content;
import java.util.ArrayList;
import java.util.List;
import java.util.ServiceLoader;
/**
* The internal
*/
public class RawMarkdownSources {
public static List<Content<?>> getAllMarkdown() {
ServiceLoader<RawMarkdownSource> loader = ServiceLoader.load(RawMarkdownSource.class);
List<Content<?>> content = new ArrayList<>();
loader.iterator().forEachRemaining(d -> content.addAll(d.getMarkdownInfo()));
return content;
}
}

10
nb-api/src/main/resources/docs-for-testing-only/topics-a/entry1-1.md Normal file
View File

@ -0,0 +1,10 @@
---
title: Entry 1-1
weight: 37
topics: entries/entry2-1, related-topic-for-entry1-1
aggregate: topic
---
# Title Heading for Entry 1-1

19
nb-api/src/test/java/io/nosqlbench/nb/api/markdown/aggregator/MarkdownDocsTest.java Normal file
View File

@ -0,0 +1,19 @@
package io.nosqlbench.nb.api.markdown.aggregator;
import org.junit.Ignore;
import org.junit.Test;
import java.util.List;
import static org.assertj.core.api.Assertions.assertThat;
public class MarkdownDocsTest {
@Test
public void testLoadMarkdown() {
List<MarkdownInfo> all = MarkdownDocs.findAll();
assertThat(all).hasSizeGreaterThan(0);
}
}

0
nb-api/src/test/resources/testmarkdown/topics-a/entry1-1.md → nb-api/src/test/resources/docs-for-testing-only/topics-a/entry1-1.md
View File